1414That means a delayed or pulled rollout is never announced as live. When a release
1515was due but has not shipped, release management get a heads-up instead.
1616
17+ The App Store is read twice before that heads-up goes out. Apple serves the lookup
18+ API with a 24 hour cache lifetime, so an edge node that answered before the rollout
19+ can report the previous version for the rest of the day and make a shipped release
20+ look late. The web listing carries the same two facts with a 15 minute lifetime, so
21+ it settles the disagreement.
22+
1723Two webhooks, set from repo secrets in the workflow:
1824 SLACK_WEBHOOK_URL announcement channel (IOS_RELEASE_SLACK_WEBHOOK)
1925 RELMAN_WEBHOOK_URL release management channel (RELMAN_SLACK_WEBHOOK), used
2733"""
2834import json
2935import os
36+ import re
3037import sys
3138import time
3239import urllib .error
3340import urllib .request
41+ from collections .abc import Callable
3442from datetime import date , datetime , timezone
43+ from typing import Any
3544
3645SCHEDULE_URL = "https://whattrainisitnow.com/api/release/schedule/?version=nightly"
3746IOS_SCHEDULE_URL = "https://whattrainisitnow.com/api/release/schedule/ios/?version={}"
38- # Firefox iOS App Store listing
39- ITUNES_URL = "https://itunes.apple.com/lookup?id=989804926&country=us"
47+ # Firefox iOS App Store listing. Unknown query parameters form part of Apple's cache
48+ # key, so filling the unused one with the current time asks for a URL no edge node
49+ # holds yet, which gets us today's answer rather than yesterday's.
50+ ITUNES_URL = "https://itunes.apple.com/lookup?id=989804926&country=us&_={}"
51+ # The same listing as a web page, read only when the lookup API disagrees with the
52+ # schedule.
53+ APPS_URL = "https://apps.apple.com/us/app/firefox-fast-private-browser/id989804926"
54+
55+ # The page ships its data as JSON in a script tag. Scraping the rendered markup instead
56+ # would mean matching class names like "svelte-1t5dyc5", which are build hashes and
57+ # rotate whenever Apple redeploys.
58+ SERVER_DATA = re .compile (
59+ r'<script type="application/json" id="serialized-server-data">(.*?)</script>' ,
60+ re .DOTALL ,
61+ )
62+ # Where the version sits in that blob. Undocumented and unversioned, hence
63+ # AppStorePageError below: a change of shape has to read as "could not confirm".
64+ VERSION_ITEM = ("data" , 0 , "data" , "shelfMapping" , "mostRecentVersion" , "items" , 0 )
65+ # secondarySubtitle is a JavaScript Date.toString(), e.g. "Mon Aug 17 2026 03:01:57
66+ # GMT+0000 (Coordinated Universal Time)". Keep the offset, drop the trailing name.
67+ JS_DATE = re .compile (r"(.+? GMT[+-]\d{4})" )
68+ JS_DATE_FORMAT = "%a %b %d %Y %H:%M:%S GMT%z"
4069
4170MESSAGE = "Firefox iOS v{version} is live with a phased rollout as of {rolled_out}"
4271
4574 "showing v{live} (live since {rolled_out}). Investigate why v{expected} is not live."
4675)
4776
77+ # Sent when the two App Store sources cannot be made to agree. It has to claim less
78+ # than MISMATCH_MESSAGE does: all we know is that a release was due and that nothing
79+ # confirmed it, which is worth a look either way.
80+ UNCONFIRMED_MESSAGE = (
81+ "Firefox iOS v{expected} was scheduled to roll out today. The App Store lookup "
82+ "shows v{live}, and that could not be checked against the App Store page "
83+ "({reason}), so whether v{expected} is live is unknown. Please check manually."
84+ )
85+
4886# The shipping iOS major is behind the current Nightly by two versions. Desktop merge day is a
4987# Thursday, so by the Monday rollout Nightly has always moved on.
5088NIGHTLY_OFFSET = 2
5593RETRIES = 3
5694BACKOFF_SECONDS = 5
5795
96+
97+ class AppStorePageError (Exception ):
98+ """The App Store page did not hold a version where one was expected."""
99+
100+
58101# HTTPError subclasses URLError, so this covers HTTP errors, connection failures
59102# and timeouts. JSONDecodeError catches an error page served in place of JSON.
60- RETRYABLE = (urllib .error .URLError , TimeoutError , json .JSONDecodeError )
103+ # A truncated page is worth another go too, so AppStorePageError joins them.
104+ RETRYABLE = (
105+ urllib .error .URLError ,
106+ TimeoutError ,
107+ json .JSONDecodeError ,
108+ AppStorePageError ,
109+ )
61110
62111# Env vars are strings, and every non-empty string is truthy, so bool("0") is True.
63112# Spell the accepted values out rather than let DRY_RUN=false enable a dry run.
@@ -75,12 +124,18 @@ def env_flag(name: str) -> bool:
75124 raise ValueError (f"{ name } must be a boolean-ish value, got { value !r} " )
76125
77126
78- def fetch_json (url : str ) -> dict :
79- """GET JSON, retrying transient failures. A 4xx other than 429 fails at once."""
127+ def fetch (url : str , parse : Callable [[Any ], Any ]) -> Any :
128+ """
129+ GET url and hand the response to parse, retrying transient failures. A 4xx other
130+ than 429 fails at once.
131+
132+ parse runs inside the retry so that a half-read or error page is retried rather
133+ than raised, which is why it takes the response instead of the caller reading it.
134+ """
80135 for attempt in range (1 , RETRIES + 1 ):
81136 try :
82137 with urllib .request .urlopen (url , timeout = TIMEOUT_SECONDS ) as resp :
83- return json . load (resp )
138+ return parse (resp )
84139 except RETRYABLE as error :
85140 # A bad URL or a rejected request will not fix itself on a retry.
86141 permanent = (
@@ -100,6 +155,11 @@ def fetch_json(url: str) -> dict:
100155 raise AssertionError ("unreachable: the loop either returns or raises" )
101156
102157
158+ def fetch_json (url : str ) -> dict :
159+ """GET JSON, retrying transient failures."""
160+ return fetch (url , json .load )
161+
162+
103163def parse_date (value : str ) -> date :
104164 """Parse API date string (e.g. '2026-07-27 02:00:00+00:00') to date object."""
105165 return datetime .strptime (value .split (" " )[0 ], "%Y-%m-%d" ).date ()
@@ -144,7 +204,7 @@ def expected_version(major: int, today: date) -> str | None:
144204
145205def live_release () -> tuple [str , datetime ]:
146206 """The version currently live on the App Store, and when its rollout began."""
147- data = fetch_json (ITUNES_URL )
207+ data = fetch_json (ITUNES_URL . format ( int ( time . time ())) )
148208 if not data .get ("results" ):
149209 raise RuntimeError ("iTunes lookup returned no results for Firefox iOS." )
150210 app = data ["results" ][0 ]
@@ -154,6 +214,63 @@ def live_release() -> tuple[str, datetime]:
154214 return app ["version" ], rolled_out
155215
156216
217+ def parse_listing (resp : Any ) -> tuple [str , datetime ]:
218+ """Pull the version and rollout time out of the App Store page's JSON blob."""
219+ page = SERVER_DATA .search (resp .read ().decode ("utf-8" , "replace" ))
220+ if not page :
221+ raise AppStorePageError ("no serialized-server-data on the App Store page" )
222+
223+ node = json .loads (page .group (1 ))
224+ try :
225+ for step in VERSION_ITEM :
226+ node = node [step ]
227+ # The same value appears with and without the prefix elsewhere in the blob.
228+ version = node ["primarySubtitle" ].removeprefix ("Version " ).strip ()
229+ when = node ["secondarySubtitle" ]
230+ except (LookupError , TypeError , AttributeError ) as error :
231+ raise AppStorePageError (f"unexpected App Store page shape: { error } " ) from error
232+
233+ stamp = JS_DATE .match (when )
234+ if not stamp :
235+ raise AppStorePageError (f"could not read a rollout date from { when !r} " )
236+ return version , datetime .strptime (stamp .group (1 ), JS_DATE_FORMAT )
237+
238+
239+ def live_release_from_page () -> tuple [str , datetime ]:
240+ """As live_release, from the web listing. Cached for minutes rather than a day."""
241+ return fetch (APPS_URL , parse_listing )
242+
243+
244+ def live_release_confirmed (expected : str ) -> tuple [str , datetime , str | None ]:
245+ """
246+ The live version and rollout time, cross-checked against the web listing whenever
247+ the lookup API disagrees with the schedule.
248+
249+ The third element is None when the answer can be trusted, otherwise why it could
250+ not be confirmed. The page is only read on disagreement: it is a large response
251+ parsed out of an undocumented blob, so there is no sense paying for it to
252+ re-confirm an answer that already matches.
253+ """
254+ live , rolled_out = live_release ()
255+ if live == expected :
256+ return live , rolled_out , None
257+
258+ # Not evidence of a delay yet — the lookup API may just be serving a cached
259+ # pre-rollout answer. Ask the page, which goes stale in minutes, before anyone's
260+ # phone buzzes.
261+ print (f"Lookup API shows { live } , expected { expected } — checking the page." )
262+ try :
263+ page_live , page_rolled_out = live_release_from_page ()
264+ except RETRYABLE as error :
265+ # Unreachable or reshaped, so we cannot tell a stale lookup API from a release
266+ # that really has not shipped. Hand back what the API said along with the
267+ # reason, so the alert can quote both.
268+ return live , rolled_out , str (error )
269+
270+ print (f"The page shows { page_live } (rolled out { format_rollout (page_rolled_out )} )." )
271+ return page_live , page_rolled_out , None
272+
273+
157274def format_rollout (when : datetime ) -> str :
158275 """Render the rollout time for Slack, e.g. '02:11 UTC on 2026-07-27'."""
159276 return f"{ when :%H:%M} UTC on { when :%Y-%m-%d} "
@@ -173,6 +290,22 @@ def post_to_slack(webhook_url: str, text: str) -> None:
173290 raise RuntimeError (f"Slack webhook returned HTTP { resp .status } " )
174291
175292
293+ def alert_relman (webhook_url : str | None , text : str , dry_run : bool , done : str ) -> bool :
294+ """Alert release management. False if the alert could not be delivered."""
295+ if dry_run :
296+ print (f"DRY RUN — would alert release management:\n { text } " )
297+ return True
298+ if not webhook_url :
299+ print (
300+ "RELMAN_WEBHOOK_URL is not set — cannot alert release management." ,
301+ file = sys .stderr ,
302+ )
303+ return False
304+ post_to_slack (webhook_url , text )
305+ print (done )
306+ return True
307+
308+
176309def main () -> int :
177310 dry_run = env_flag ("DRY_RUN" )
178311 webhook_url = os .environ .get ("SLACK_WEBHOOK_URL" )
@@ -194,26 +327,39 @@ def main() -> int:
194327 if expected is None :
195328 return 0
196329
197- live , rolled_out = live_release ()
330+ live , rolled_out , unconfirmed = live_release_confirmed (expected )
331+ if unconfirmed :
332+ # Neither source could settle it. Say exactly that and let a human decide,
333+ # rather than going quiet on a release that was due today.
334+ print (
335+ f"{ today } : could not confirm iOS { expected } : { unconfirmed } " ,
336+ file = sys .stderr ,
337+ )
338+ alert_relman (
339+ relman_webhook_url ,
340+ UNCONFIRMED_MESSAGE .format (
341+ expected = expected , live = live , reason = unconfirmed
342+ ),
343+ dry_run ,
344+ f"Asked release management to check iOS { expected } manually." ,
345+ )
346+ # Non-zero regardless: the check itself broke and wants fixing.
347+ return 1
348+
198349 if live != expected :
199350 # Scheduled but not live: rollout delayed, pulled, or the version we
200351 # derived is wrong. Release management want to know either way.
201352 print (f"{ today } : iOS { expected } was due today but the App Store shows { live } ." )
202353 alert = MISMATCH_MESSAGE .format (
203354 expected = expected , live = live , rolled_out = format_rollout (rolled_out )
204355 )
205- if dry_run :
206- print (f"DRY RUN — would alert release management:\n { alert } " )
207- return 0
208- if not relman_webhook_url :
209- print (
210- "RELMAN_WEBHOOK_URL is not set — cannot alert release management." ,
211- file = sys .stderr ,
212- )
213- return 1
214- post_to_slack (relman_webhook_url , alert )
215- print (f"Alerted release management that iOS { expected } has not shipped." )
216- return 0
356+ delivered = alert_relman (
357+ relman_webhook_url ,
358+ alert ,
359+ dry_run ,
360+ f"Alerted release management that iOS { expected } has not shipped." ,
361+ )
362+ return 0 if delivered else 1
217363
218364 text = MESSAGE .format (version = live , rolled_out = format_rollout (rolled_out ))
219365 if dry_run :
0 commit comments