Hi, up until now I’ve used to use a little Python script:
urls = [
f"https://www.unrealengine.com/en-US/blog/featured-free-marketplace-content-{monthAsString}-{yearAs4numbers}",
f"https://www.unrealengine.com/en-US/blog/featured-free-unreal-marketplace-content-{monthAsString}-{yearAs4numbers}",
f"https://www.unrealengine.com/en-US/blog/featured-free-unreal-engine-marketplace-content---{monthAsString}-{yearAs4numbers}",
f"https://www.unrealengine.com/en-US/blog/featured-free-unreal-engine-marketplace-content-{monthAsString}-{yearAs4numbers}",
]
for url in urls:
url = url.lower()
print(f"url: {url}")
try:
# FeaturedMarketplaceContentLastSentURLAsStr = replit.db["FeaturedMarketplaceContentLastSentURLAsStr"]
FeaturedMarketplaceContentLastSentURLAsStr = unrealCommunityDb.FeaturedMarketplaceContentLastSentURLAsStr
if FeaturedMarketplaceContentLastSentURLAsStr == None:
raise Exception
except:
# replit.db["FeaturedMarketplaceContentLastSentURLAsStr"] = "None"
unrealCommunityDb.FeaturedMarketplaceContentLastSentURLAsStr = "None"
FeaturedMarketplaceContentLastSentURLAsStr = "None"
hasEpicPublishedPage = False
from urllib import error as urllibError
try:
try:
fp: http.client.HTTPResponse = urllib.request.urlopen(urllib.request.Request(url,headers={'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept-Language': 'en-US,en;q=0.9', 'Sec-Ch-Ua': '"Microsoft Edge";v="123", "Not:A-Brand";v="8", "Chromium";v="123"', 'Sec-Ch-Ua-Mobile': '?0', 'Sec-Ch-Ua-Platform': '"Windows"', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0'}))
except urllibError.HTTPError as exc:
print(f"error opening url: {traceback.format_exc()}")
else:
print(fp.status)
mybytes = fp.read()
webpageHtml:str = mybytes.decode("utf8")
fp.close()
# print(webpageHtml)
if fp.status == 200:
hasEpicPublishedPage = True
# print(f"hasEpicPublishedPage: {hasEpicPublishedPage}")
# print(f"unrealCommunityDb.FeaturedMarketplaceContentLastSentURLAsStr: {unrealCommunityDb.FeaturedMarketplaceContentLastSentURLAsStr}")
# print(f"FeaturedMarketplaceContentLastSentURLAsStr: {FeaturedMarketplaceContentLastSentURLAsStr}")
# print(f"monthAsString: {monthAsString}")
# print(f"monthAsString: {yearAs4numbers}")
# print(f"expression: {not (FeaturedMarketplaceContentLastSentURLAsStr.__contains__(monthAsString) and FeaturedMarketplaceContentLastSentURLAsStr.__contains__(yearAs4numbers))}")
except Exception as exception:
print(traceback.format_exc())
pass
# if hasEpicPublishedPage and url != FeaturedMarketplaceContentLastSentURLAsStr:
if hasEpicPublishedPage and not (FeaturedMarketplaceContentLastSentURLAsStr.__contains__(monthAsString) and FeaturedMarketplaceContentLastSentURLAsStr.__contains__(str(yearAs4numbers))):
print("mhm2.1")
unrealCommunityDb.FeaturedMarketplaceContentLastSentURLAsStr = url
# print("mhm2.2")
# replit.db["FeaturedMarketplaceContentLastSentURLAsStr"] = url
for guild in client.guilds:
# print("mhm3")
# if guild.name.lower().__contains__('unreal engine'):
if True:
# print("mhm4")
announcementsChannel = Helpers.getAnnouncementsChannel(guild)
if not announcementsChannel:
continue
print("Sending monthly free marketplace content msg")
# await Helpers.createAndSendEmbed(f"New featured free marketplace content is now available! {url} @ChannelBot <@978644028599980055>", announcementsChannel)
# await Helpers.createAndSendEmbed(f"<@1004830823465361588> New featured free marketplace content is now available! @ChannelBot <@&990046126851915806>", announcementsChannel, withEmbeddedServerIcon=True, linkUrl=url, title=f"Featured free marketplace content—{monthAsString.capitalize()} {yearAs4numbers}")
await Helpers.sendMessage(f"<@&1004830823465361588> {monthAsString.capitalize()}'s free marketplace content is now available, check it out! {url}", announcementsChannel,supressAllMentions=False)
# await Helpers.createAndSendEmbed(f"<@1004830823465361588> New featured free marketplace content is now available! @ChannelBot <@&990046126851915806>", announcementsChannel, linkUrl=url)
# <@979084167172214814> @Everything role
(sorry its a little messy, the TLDR is that it makes a guess on what the marketplace URL will be based on the current month and year and then checks if the URL responds with 200 OK - its a copy pasta from a longer script so it will need some modifications if you want to try it for youself).
Since 2 months back however, cloudflare have started responding with HTTP/1.1 403 Forbidden ... cf-mitigated: challenge. So it no longer works for figuring out when the new url is available.
Im guessing theres no chance we could get some sort of API for reading the blogposts?
The good old RSS feed at https://unrealengine.com/rss is also down - so I cant seem to use that either
I ping 100s of people every month in a Discord server. I would like to be able to automatically verify the url before I have my bot sending anything.
