Skip to content

Commit

Permalink
Fix: point latest.html to stable and testing.html to testing for rele…
Browse files Browse the repository at this point in the history 
…ases
  • Loading branch information
@TrueBrain
TrueBrain committed Feb 9, 2019
1 parent 33fb048 commit 766d2dd
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 49 deletions.
4 changes: 2 additions & 2 deletions _layouts/default.html
View file
Expand Up @@ -20,8 +20,8 @@
</head>
<body>
<div id="header">
{% assign latest_stable = site.downloads | where_exp: "download", "download.id contains '/downloads/openttd-releases/'" | where_exp: "download", "download.name == 'stable'" | sort: "id" | last %}
{% assign latest_testing = site.downloads | where_exp: "download", "download.id == '/downloads/openttd-releases/latest'" | last %}
{% assign latest_stable = site.downloads | where_exp: "download", "download.id == '/downloads/openttd-releases/latest'" | last %}
{% assign latest_testing = site.downloads | where_exp: "download", "download.id == '/downloads/openttd-releases/testing'" | last %}
{% assign latest_nightly = site.downloads | where_exp: "download", "download.id == '/downloads/openttd-nightlies/latest'" | last %}

{% if latest_nightly.version == latest_stable.version or latest_nightly.version == latest_testing.version %}
Expand Down
101 changes: 55 additions & 46 deletions fetch-downloads.py
View file
Expand Up @@ -24,13 +24,11 @@
}

# Current supported types on the new infrastructure
types = [
"openttd-nightlies",
"openttd-releases",
]
types_grouped_by_branch = [
"openttd-pullrequests",
]
types = {
"openttd-nightlies": "flatten",
"openttd-releases": "stable-testing",
"openttd-pullrequests": "name",
}


async def download(url):
Expand Down Expand Up @@ -103,15 +101,6 @@ async def get_old_versions_of_folder(folder):
return versions


async def get_latest(folder):
# We use the non-CDN URL here, as we want the latest; not any edge-cached version
latest = await download(f"https://openttd.ams3.digitaloceanspaces.com/{folder}/latest.txt")
latest = latest.decode()

version, _, date = latest.partition(",")
return version


async def get_listing(folder):
# We use the non-CDN URL here, as we want the latest; not any edge-cached version
listing = await download(f"https://openttd.ams3.digitaloceanspaces.com/{folder}/listing.txt")
Expand All @@ -121,8 +110,8 @@ async def get_listing(folder):
if not entry:
continue

version, _, date = entry.partition(",")
versions[version] = date
version, date, name = entry.split(",", 3)
versions[version] = (date, name)

return versions

Expand All @@ -146,8 +135,9 @@ async def write_to_collection(type, version, manifest):
f.write("---\n".encode())


async def handle_version(folder, type, version, host, on_old_infrastructure="false", latest=None):
print(f"Adding {type}/{version} to downloads collection ..")
async def handle_version(folder, dest_folder, version, host, on_old_infrastructure="false",
latest=None, latest_filenames=None):
print(f"Adding {dest_folder}/{version} to downloads collection ..")
manifest = await fetch_manifest(folder, version, host, on_old_infrastructure=on_old_infrastructure)

# Insert the version into the header; otherwise we don't know
Expand All @@ -156,11 +146,15 @@ async def handle_version(folder, type, version, host, on_old_infrastructure="fal
manifest.insert(1, f"version: {version}")
manifest = "\n".join(manifest)

await write_to_collection(type, version, manifest)
await write_to_collection(dest_folder, version, manifest)

# If this is the current version, also copy the content to "latest"
if version == latest:
await write_to_collection(type, "latest", manifest)
if latest_filenames is None:
latest_filenames = ["latest"]

for latest_filename in latest_filenames:
await write_to_collection(dest_folder, latest_filename, manifest)


async def main():
Expand Down Expand Up @@ -191,41 +185,56 @@ async def main():
latest=latest)

# Support for new infrastructure; those are hosted on the DigitalOcean CDN (Spaces)
for type in types:
latest = await get_latest(type)
versions = await get_listing(type)
for version, data in versions.items():
os.makedirs(f"_downloads/{type}", exist_ok=True)
await handle_version(
type,
type,
version,
"https://proxy.binaries.openttd.org",
latest=latest)

for type in types_grouped_by_branch:
for type, method in types.items():
version_grouped = defaultdict(list)
latest_grouped = defaultdict(str)
latest_grouped = defaultdict(lambda: ["", ""])
versions = await get_listing(type)

# Regroup the versions based on the branch (which is the second part in the name)
# Regroup the versions based on the method
for version, data in versions.items():
(date, branch, tag) = version.split("-", 3)
(date, name) = data

if method == "flatten":
index = None
else:
index = name

version_grouped[branch].append((version, data))
if latest_grouped[branch] < date:
latest_grouped[branch] = version
version_grouped[index].append(version)
if latest_grouped[index][0] < date:
latest_grouped[index] = (date, version)

# Now, based on the group, generate the files
for branch in version_grouped.keys():
for version, data in version_grouped[branch]:
os.makedirs(f"_downloads/{type}/{branch}", exist_ok=True)
for index in version_grouped.keys():
for version in version_grouped[index]:
if method == "name":
dest_folder = f"{type}/{index}"
else:
dest_folder = type

if method == "stable-testing":
if index == "stable":
# Force 'latest' to always point to 'stable'
latest_filenames = ["latest"]
else:
latest_filenames = ["testing"]

# In case testing is older than stable, stable becomes the testing
if latest_grouped["testing"][0] < latest_grouped["stable"][0]:
if index == "stable":
latest_filenames.append("testing")
else:
latest_filenames = []
else:
latest_filenames = ["latest"]

os.makedirs(f"_downloads/{dest_folder}", exist_ok=True)
await handle_version(
type,
f"{type}/{branch}",
dest_folder,
version,
"https://proxy.binaries.openttd.org",
latest=latest_grouped[branch])
latest=latest_grouped[index][1],
latest_filenames=latest_filenames)

await session.close()

Expand Down
2 changes: 1 addition & 1 deletion nginx.default.conf
View file
Expand Up @@ -79,7 +79,7 @@ server {

# Download pages of the latest version
location ~ ^/(\w\w/)?download-stable$ { return 301 $scheme://$http_host/downloads/openttd-releases/latest.html; }
location ~ ^/(\w\w/)?download-testing$ { return 301 $scheme://$http_host/downloads/openttd-releases/latest.html; }
location ~ ^/(\w\w/)?download-testing$ { return 301 $scheme://$http_host/downloads/openttd-releases/testing.html; }
location ~ ^/(\w\w/)?download-trunk$ { return 301 $scheme://$http_host/downloads/openttd-nightlies/latest.html; }
location ~ ^/(\w\w/)?download-catcodec$ { return 301 $scheme://$http_host/downloads/catcodec-releases/latest.html; }
location ~ ^/(\w\w/)?download-grfcodec$ { return 301 $scheme://$http_host/downloads/grfcodec-releases/latest.html; }
Expand Down

0 comments on commit 766d2dd

Please sign in to comment.