Skip to content

Commit 0220462

Browse files
committed
Add an fallback to check the app page listing in ios_release_slack_notifier
1 parent 2a65bf0 commit 0220462

1 file changed

Lines changed: 166 additions & 20 deletions

File tree

‎app/scripts/ios_release_slack_notifier.py‎

Lines changed: 166 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@
1414
That means a delayed or pulled rollout is never announced as live. When a release
1515
was due but has not shipped, release management get a heads-up instead.
1616
17+
The App Store is read twice before that heads-up goes out. Apple serves the lookup
18+
API with a 24 hour cache lifetime, so an edge node that answered before the rollout
19+
can report the previous version for the rest of the day and make a shipped release
20+
look late. The web listing carries the same two facts with a 15 minute lifetime, so
21+
it settles the disagreement.
22+
1723
Two webhooks, set from repo secrets in the workflow:
1824
SLACK_WEBHOOK_URL announcement channel (IOS_RELEASE_SLACK_WEBHOOK)
1925
RELMAN_WEBHOOK_URL release management channel (RELMAN_SLACK_WEBHOOK), used
@@ -27,16 +33,39 @@
2733
"""
2834
import json
2935
import os
36+
import re
3037
import sys
3138
import time
3239
import urllib.error
3340
import urllib.request
41+
from collections.abc import Callable
3442
from datetime import date, datetime, timezone
43+
from typing import Any
3544

3645
SCHEDULE_URL = "https://whattrainisitnow.com/api/release/schedule/?version=nightly"
3746
IOS_SCHEDULE_URL = "https://whattrainisitnow.com/api/release/schedule/ios/?version={}"
38-
# Firefox iOS App Store listing
39-
ITUNES_URL = "https://itunes.apple.com/lookup?id=989804926&country=us"
47+
# Firefox iOS App Store listing. Unknown query parameters form part of Apple's cache
48+
# key, so filling the unused one with the current time asks for a URL no edge node
49+
# holds yet, which gets us today's answer rather than yesterday's.
50+
ITUNES_URL = "https://itunes.apple.com/lookup?id=989804926&country=us&_={}"
51+
# The same listing as a web page, read only when the lookup API disagrees with the
52+
# schedule.
53+
APPS_URL = "https://apps.apple.com/us/app/firefox-fast-private-browser/id989804926"
54+
55+
# The page ships its data as JSON in a script tag. Scraping the rendered markup instead
56+
# would mean matching class names like "svelte-1t5dyc5", which are build hashes and
57+
# rotate whenever Apple redeploys.
58+
SERVER_DATA = re.compile(
59+
r'<script type="application/json" id="serialized-server-data">(.*?)</script>',
60+
re.DOTALL,
61+
)
62+
# Where the version sits in that blob. Undocumented and unversioned, hence
63+
# AppStorePageError below: a change of shape has to read as "could not confirm".
64+
VERSION_ITEM = ("data", 0, "data", "shelfMapping", "mostRecentVersion", "items", 0)
65+
# secondarySubtitle is a JavaScript Date.toString(), e.g. "Mon Aug 17 2026 03:01:57
66+
# GMT+0000 (Coordinated Universal Time)". Keep the offset, drop the trailing name.
67+
JS_DATE = re.compile(r"(.+? GMT[+-]\d{4})")
68+
JS_DATE_FORMAT = "%a %b %d %Y %H:%M:%S GMT%z"
4069

4170
MESSAGE = "Firefox iOS v{version} is live with a phased rollout as of {rolled_out}"
4271

@@ -45,6 +74,15 @@
4574
"showing v{live} (live since {rolled_out}). Investigate why v{expected} is not live."
4675
)
4776

77+
# Sent when the two App Store sources cannot be made to agree. It has to claim less
78+
# than MISMATCH_MESSAGE does: all we know is that a release was due and that nothing
79+
# confirmed it, which is worth a look either way.
80+
UNCONFIRMED_MESSAGE = (
81+
"Firefox iOS v{expected} was scheduled to roll out today. The App Store lookup "
82+
"shows v{live}, and that could not be checked against the App Store page "
83+
"({reason}), so whether v{expected} is live is unknown. Please check manually."
84+
)
85+
4886
# The shipping iOS major is behind the current Nightly by two versions. Desktop merge day is a
4987
# Thursday, so by the Monday rollout Nightly has always moved on.
5088
NIGHTLY_OFFSET = 2
@@ -55,9 +93,20 @@
5593
RETRIES = 3
5694
BACKOFF_SECONDS = 5
5795

96+
97+
class AppStorePageError(Exception):
98+
"""The App Store page did not hold a version where one was expected."""
99+
100+
58101
# HTTPError subclasses URLError, so this covers HTTP errors, connection failures
59102
# and timeouts. JSONDecodeError catches an error page served in place of JSON.
60-
RETRYABLE = (urllib.error.URLError, TimeoutError, json.JSONDecodeError)
103+
# A truncated page is worth another go too, so AppStorePageError joins them.
104+
RETRYABLE = (
105+
urllib.error.URLError,
106+
TimeoutError,
107+
json.JSONDecodeError,
108+
AppStorePageError,
109+
)
61110

62111
# Env vars are strings, and every non-empty string is truthy, so bool("0") is True.
63112
# Spell the accepted values out rather than let DRY_RUN=false enable a dry run.
@@ -75,12 +124,18 @@ def env_flag(name: str) -> bool:
75124
raise ValueError(f"{name} must be a boolean-ish value, got {value!r}")
76125

77126

78-
def fetch_json(url: str) -> dict:
79-
"""GET JSON, retrying transient failures. A 4xx other than 429 fails at once."""
127+
def fetch(url: str, parse: Callable[[Any], Any]) -> Any:
128+
"""
129+
GET url and hand the response to parse, retrying transient failures. A 4xx other
130+
than 429 fails at once.
131+
132+
parse runs inside the retry so that a half-read or error page is retried rather
133+
than raised, which is why it takes the response instead of the caller reading it.
134+
"""
80135
for attempt in range(1, RETRIES + 1):
81136
try:
82137
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as resp:
83-
return json.load(resp)
138+
return parse(resp)
84139
except RETRYABLE as error:
85140
# A bad URL or a rejected request will not fix itself on a retry.
86141
permanent = (
@@ -100,6 +155,11 @@ def fetch_json(url: str) -> dict:
100155
raise AssertionError("unreachable: the loop either returns or raises")
101156

102157

158+
def fetch_json(url: str) -> dict:
159+
"""GET JSON, retrying transient failures."""
160+
return fetch(url, json.load)
161+
162+
103163
def parse_date(value: str) -> date:
104164
"""Parse API date string (e.g. '2026-07-27 02:00:00+00:00') to date object."""
105165
return datetime.strptime(value.split(" ")[0], "%Y-%m-%d").date()
@@ -144,7 +204,7 @@ def expected_version(major: int, today: date) -> str | None:
144204

145205
def live_release() -> tuple[str, datetime]:
146206
"""The version currently live on the App Store, and when its rollout began."""
147-
data = fetch_json(ITUNES_URL)
207+
data = fetch_json(ITUNES_URL.format(int(time.time())))
148208
if not data.get("results"):
149209
raise RuntimeError("iTunes lookup returned no results for Firefox iOS.")
150210
app = data["results"][0]
@@ -154,6 +214,63 @@ def live_release() -> tuple[str, datetime]:
154214
return app["version"], rolled_out
155215

156216

217+
def parse_listing(resp: Any) -> tuple[str, datetime]:
218+
"""Pull the version and rollout time out of the App Store page's JSON blob."""
219+
page = SERVER_DATA.search(resp.read().decode("utf-8", "replace"))
220+
if not page:
221+
raise AppStorePageError("no serialized-server-data on the App Store page")
222+
223+
node = json.loads(page.group(1))
224+
try:
225+
for step in VERSION_ITEM:
226+
node = node[step]
227+
# The same value appears with and without the prefix elsewhere in the blob.
228+
version = node["primarySubtitle"].removeprefix("Version ").strip()
229+
when = node["secondarySubtitle"]
230+
except (LookupError, TypeError, AttributeError) as error:
231+
raise AppStorePageError(f"unexpected App Store page shape: {error}") from error
232+
233+
stamp = JS_DATE.match(when)
234+
if not stamp:
235+
raise AppStorePageError(f"could not read a rollout date from {when!r}")
236+
return version, datetime.strptime(stamp.group(1), JS_DATE_FORMAT)
237+
238+
239+
def live_release_from_page() -> tuple[str, datetime]:
240+
"""As live_release, from the web listing. Cached for minutes rather than a day."""
241+
return fetch(APPS_URL, parse_listing)
242+
243+
244+
def live_release_confirmed(expected: str) -> tuple[str, datetime, str | None]:
245+
"""
246+
The live version and rollout time, cross-checked against the web listing whenever
247+
the lookup API disagrees with the schedule.
248+
249+
The third element is None when the answer can be trusted, otherwise why it could
250+
not be confirmed. The page is only read on disagreement: it is a large response
251+
parsed out of an undocumented blob, so there is no sense paying for it to
252+
re-confirm an answer that already matches.
253+
"""
254+
live, rolled_out = live_release()
255+
if live == expected:
256+
return live, rolled_out, None
257+
258+
# Not evidence of a delay yet — the lookup API may just be serving a cached
259+
# pre-rollout answer. Ask the page, which goes stale in minutes, before anyone's
260+
# phone buzzes.
261+
print(f"Lookup API shows {live}, expected {expected} — checking the page.")
262+
try:
263+
page_live, page_rolled_out = live_release_from_page()
264+
except RETRYABLE as error:
265+
# Unreachable or reshaped, so we cannot tell a stale lookup API from a release
266+
# that really has not shipped. Hand back what the API said along with the
267+
# reason, so the alert can quote both.
268+
return live, rolled_out, str(error)
269+
270+
print(f"The page shows {page_live} (rolled out {format_rollout(page_rolled_out)}).")
271+
return page_live, page_rolled_out, None
272+
273+
157274
def format_rollout(when: datetime) -> str:
158275
"""Render the rollout time for Slack, e.g. '02:11 UTC on 2026-07-27'."""
159276
return f"{when:%H:%M} UTC on {when:%Y-%m-%d}"
@@ -173,6 +290,22 @@ def post_to_slack(webhook_url: str, text: str) -> None:
173290
raise RuntimeError(f"Slack webhook returned HTTP {resp.status}")
174291

175292

293+
def alert_relman(webhook_url: str | None, text: str, dry_run: bool, done: str) -> bool:
294+
"""Alert release management. False if the alert could not be delivered."""
295+
if dry_run:
296+
print(f"DRY RUN — would alert release management:\n{text}")
297+
return True
298+
if not webhook_url:
299+
print(
300+
"RELMAN_WEBHOOK_URL is not set — cannot alert release management.",
301+
file=sys.stderr,
302+
)
303+
return False
304+
post_to_slack(webhook_url, text)
305+
print(done)
306+
return True
307+
308+
176309
def main() -> int:
177310
dry_run = env_flag("DRY_RUN")
178311
webhook_url = os.environ.get("SLACK_WEBHOOK_URL")
@@ -194,26 +327,39 @@ def main() -> int:
194327
if expected is None:
195328
return 0
196329

197-
live, rolled_out = live_release()
330+
live, rolled_out, unconfirmed = live_release_confirmed(expected)
331+
if unconfirmed:
332+
# Neither source could settle it. Say exactly that and let a human decide,
333+
# rather than going quiet on a release that was due today.
334+
print(
335+
f"{today}: could not confirm iOS {expected}: {unconfirmed}",
336+
file=sys.stderr,
337+
)
338+
alert_relman(
339+
relman_webhook_url,
340+
UNCONFIRMED_MESSAGE.format(
341+
expected=expected, live=live, reason=unconfirmed
342+
),
343+
dry_run,
344+
f"Asked release management to check iOS {expected} manually.",
345+
)
346+
# Non-zero regardless: the check itself broke and wants fixing.
347+
return 1
348+
198349
if live != expected:
199350
# Scheduled but not live: rollout delayed, pulled, or the version we
200351
# derived is wrong. Release management want to know either way.
201352
print(f"{today}: iOS {expected} was due today but the App Store shows {live}.")
202353
alert = MISMATCH_MESSAGE.format(
203354
expected=expected, live=live, rolled_out=format_rollout(rolled_out)
204355
)
205-
if dry_run:
206-
print(f"DRY RUN — would alert release management:\n{alert}")
207-
return 0
208-
if not relman_webhook_url:
209-
print(
210-
"RELMAN_WEBHOOK_URL is not set — cannot alert release management.",
211-
file=sys.stderr,
212-
)
213-
return 1
214-
post_to_slack(relman_webhook_url, alert)
215-
print(f"Alerted release management that iOS {expected} has not shipped.")
216-
return 0
356+
delivered = alert_relman(
357+
relman_webhook_url,
358+
alert,
359+
dry_run,
360+
f"Alerted release management that iOS {expected} has not shipped.",
361+
)
362+
return 0 if delivered else 1
217363

218364
text = MESSAGE.format(version=live, rolled_out=format_rollout(rolled_out))
219365
if dry_run:

0 commit comments

Comments
 (0)