|
64717
|
2266
|
9
|
2026-05-20T14:15:00.717505+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286500717_m1.jpg...
|
Slack
|
Toast (DM) - Jiminny Inc - 4 new items - Slack
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
Mark as unread
Clear
More actions
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"All, 4 unread","depth":19,"bounds":{"left":0.56041664,"top":0.13222222,"width":0.045138888,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXRadioButton","text":"DMs","depth":19,"bounds":{"left":0.60486114,"top":0.13222222,"width":0.03125,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Add new tab","depth":20,"bounds":{"left":0.64375,"top":0.13222222,"width":0.011805556,"height":0.036666665},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Nikolay Yankov","depth":23,"bounds":{"left":0.6,"top":0.25444445,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.25444445,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.6,"top":0.25444445,"width":0.057638887,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.2777778,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12 mins","depth":23,"bounds":{"left":0.69722223,"top":0.25777778,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":24,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Jira Cloud","depth":23,"bounds":{"left":0.6,"top":0.39,"width":0.045833334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.41333333,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"20 mins","depth":23,"bounds":{"left":0.69722223,"top":0.39333335,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":24,"bounds":{"left":0.5847222,"top":0.43888888,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova transitioned a Story you are assigned to from","depth":24,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5875,"top":0.43888888,"width":0.007638889,"height":0.02}},{"char_start":1,"char_count":61,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"In QA","depth":24,"bounds":{"left":0.7125,"top":0.46333334,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"⟶","depth":24,"bounds":{"left":0.5694444,"top":0.46333334,"width":0.18055555,"height":0.044444446},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ready To Merge","depth":24,"bounds":{"left":0.5833333,"top":0.48777777,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"…","depth":24,"bounds":{"left":0.65694445,"top":0.48777777,"width":0.008333334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Review Toast","depth":23,"bounds":{"left":0.6,"top":0.52555555,"width":0.059722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.54888886,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"35 mins","depth":23,"bounds":{"left":0.69722223,"top":0.5288889,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":24,"bounds":{"left":0.5694444,"top":0.5744445,"width":0.17847222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reminder","depth":23,"bounds":{"left":0.6,"top":0.63666666,"width":0.04375,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.66,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Mark as unread","depth":24,"bounds":{"left":0.66180557,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Clear","depth":24,"bounds":{"left":0.6840278,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"More actions","depth":24,"bounds":{"left":0.70625,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Nikolay Yankov:","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако искаш виж на планетата","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.18125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":25,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.10486111,"height":0.02}}],"role_description":"text"},{"role":"AXButton","text":"Mark complete","depth":24,"bounds":{"left":0.5694444,"top":0.7211111,"width":0.08958333,"height":0.031111112},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":true,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Aneliya Angelova","depth":23,"bounds":{"left":0.6,"top":0.7922222,"width":0.056944445,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.7922222,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":15,"bounds":{"left":0.6,"top":0.7922222,"width":0.059722222,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.8155556,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.6888889,"top":0.79555553,"width":0.039583333,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"oki","depth":24,"bounds":{"left":0.5694444,"top":0.8411111,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":22,"bounds":{"left":0.6402778,"top":0.9166667,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"bounds":{"left":0.6,"top":0.97888887,"width":0.060416665,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reacted in","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.69027776,"top":0.9822222,"width":0.038194444,"height":0.012222222},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXCheckBox","text":"1 reaction, react with white check mark emoji","depth":24,"on_screen":false,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"1","depth":25,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Или си продължаваме същия тайм лайн","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Лукаш, оправи ли се с мейла?","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Home","depth":17,"bounds":{"left":0.7881944,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Messages","depth":17,"bounds":{"left":0.83194447,"top":0.12888889,"width":0.039583333,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Messages","depth":19,"bounds":{"left":0.83194447,"top":0.14,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"About","depth":17,"bounds":{"left":0.8909722,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Jump to date","depth":24,"bounds":{"left":0.8576389,"top":0.17555556,"width":0.05277778,"height":0.031111112},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Review Toast","depth":25,"bounds":{"left":0.8159722,"top":0.16,"width":0.061805554,"height":0.0011111111},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"APP","depth":25,"bounds":{"left":0.8826389,"top":0.16,"width":0.013194445,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":25,"bounds":{"left":0.8979167,"top":0.16,"width":0.0055555557,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Today at 4:38:01 PM","depth":25,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:38 PM","depth":26,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @nikolaybiaivanov","depth":26,"bounds":{"left":0.8159722,"top":0.16,"width":0.10763889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.060416665,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.8576389,"top":0.16,"width":0.013888889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"18 commits・29 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.16,"width":0.07430556,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.027083334,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20835","depth":28,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20835","depth":29,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Deployment notes:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.057638887,"height":0.027777778},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":16,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.057638887,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"• None","depth":28,"bounds":{"left":0.82708335,"top":0.20111111,"width":0.034027778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.26555556,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.26555556,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXLink","text":"Today at 4:40:26 PM","depth":26,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:40","depth":27,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":26,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":36,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":21,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446}}],"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.40111113,"width":0.068055555,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.40222222,"width":0.06527778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12106 Jy 20808 schedule reset on low queue","depth":28,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12106 Jy 20808 schedule reset on low queue","depth":29,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":42,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889}}],"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.875,"top":0.47555557,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.49888888,"width":0.06666667,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.5,"width":0.06388889,"height":0.02},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8284722,"top":0.5,"width":0.009027778,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.8375,"top":0.5,"width":0.05486111,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"33 commits・17 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":26,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.5822222,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20808","depth":28,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20808","depth":29,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"|","depth":28,"bounds":{"left":0.8979167,"top":0.5822222,"width":0.0055555557,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20928","depth":28,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20928","depth":29,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Notes:","depth":28,"bounds":{"left":0.82708335,"top":0.6311111,"width":0.030555556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Alter the selection algorithm:","depth":28,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":29,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"","depth":28,"bounds":{"left":0.87569445,"top":0.6888889,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Always take priority list FIRST.","depth":28,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":31,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08055556,"height":0.044444446}}],"role_description":"text"},{"role":"AXButton","text":"Show more","depth":27,"bounds":{"left":0.82708335,"top":0.7688889,"width":0.052083332,"height":0.024444444},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.83555555,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.83555555,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXCheckBox","text":"React with white_check_mark","depth":27,"bounds":{"left":0.8048611,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with eyes","depth":27,"bounds":{"left":0.82708335,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with raised_hands","depth":27,"bounds":{"left":0.84930557,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Add reaction…","depth":27,"bounds":{"left":0.8715278,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Reply in thread","depth":27,"bounds":{"left":0.89375,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Forward message…","depth":27,"bounds":{"left":0.91597223,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"Save for later","depth":27,"bounds":{"left":0.93819445,"top":0.26333332,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"More actions","depth":27,"on_screen":false,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"","depth":24,"bounds":{"left":0.7861111,"top":0.88,"width":0.19583334,"height":0.04222222},"on_screen":true,"value":"","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"}]...
|
-6681153539334674292
|
-8571267530870445502
|
visual_change
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
Mark as unread
Clear
More actions
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…
SlackFileEditViewGoHistoryWindowHelpDOCKER₴81DEV (-zsh)APP$82Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-cb••HomeDMsActivityFilesLater..•MoreablAl chapter • 46 m left100% <2*8•Wed 20 May 17:15:00EDQDescribe what you are looking forActivityAllDMs+C9 UnreadsNikolay Yan..DM12 minsаз сьм готов с моята част, мести ги вQА като видиш коментаритеJira Cloud20 minsApp_ Aneliya Angelova transitioned aStory you are assigned to from In QA...Review Toast35 mins::: AppPR review requested by @Vasil-JiminnyReminderDMNikolay Yankov: ако искаш виж на пл...v Mark completeAneliya An...• DM12:19 PM 5okiYesterdayWea Stovan TanevYecterdavToastXHomeMessagesAboutnotes:Today ~• Nonejiminny/appAdded by Toast for GitHubPR review requested by @Vasil-JiminnyPR reviewrequested by@Vasil Vasilev#12106 Jy 20808schedule reset onlow queue by@Vasil Vasilev37 tles chane17 files changedJIRA: JY-20808 |JY-20928Notes:Alter the selectionalgorithm:Always takepriority list FIRST.Show morejiminny/appAdded by Toast for GitHubMessage Toast+Aa..•...
|
64716
|
NULL
|
NULL
|
NULL
|
|
64716
|
2266
|
8
|
2026-05-20T14:14:59.099651+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286499099_m1.jpg...
|
Slack
|
Toast (DM) - Jiminny Inc - 4 new items - Slack
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
Reschedule
Mark as read
Clear
More actions
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"All, 4 unread","depth":19,"bounds":{"left":0.56041664,"top":0.13222222,"width":0.045138888,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXRadioButton","text":"DMs","depth":19,"bounds":{"left":0.60486114,"top":0.13222222,"width":0.03125,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Add new tab","depth":20,"bounds":{"left":0.64375,"top":0.13222222,"width":0.011805556,"height":0.036666665},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Nikolay Yankov","depth":23,"bounds":{"left":0.6,"top":0.25444445,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.25444445,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.6,"top":0.25444445,"width":0.057638887,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.2777778,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12 mins","depth":23,"bounds":{"left":0.69722223,"top":0.25777778,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":24,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Jira Cloud","depth":23,"bounds":{"left":0.6,"top":0.39,"width":0.045833334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.41333333,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"20 mins","depth":23,"bounds":{"left":0.69722223,"top":0.39333335,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":24,"bounds":{"left":0.5847222,"top":0.43888888,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova transitioned a Story you are assigned to from","depth":24,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5875,"top":0.43888888,"width":0.007638889,"height":0.02}},{"char_start":1,"char_count":61,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"In QA","depth":24,"bounds":{"left":0.7125,"top":0.46333334,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"⟶","depth":24,"bounds":{"left":0.5694444,"top":0.46333334,"width":0.18055555,"height":0.044444446},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ready To Merge","depth":24,"bounds":{"left":0.5833333,"top":0.48777777,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"…","depth":24,"bounds":{"left":0.65694445,"top":0.48777777,"width":0.008333334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Review Toast","depth":23,"bounds":{"left":0.6,"top":0.52555555,"width":0.059722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.54888886,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"35 mins","depth":23,"bounds":{"left":0.69722223,"top":0.5288889,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":24,"bounds":{"left":0.5694444,"top":0.5744445,"width":0.17847222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reminder","depth":23,"bounds":{"left":0.6,"top":0.63666666,"width":0.038194444,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.63666666,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":7,"bounds":{"left":0.6,"top":0.63666666,"width":0.036111113,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.66,"width":0.015972223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXPopUpButton","text":"Reschedule","depth":24,"bounds":{"left":0.63958335,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Mark as read","depth":24,"bounds":{"left":0.66180557,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Clear","depth":24,"bounds":{"left":0.6840278,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"More actions","depth":24,"bounds":{"left":0.70625,"top":0.63,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Nikolay Yankov:","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако искаш виж на планетата","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.18125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":25,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.10486111,"height":0.02}}],"role_description":"text"},{"role":"AXButton","text":"Mark complete","depth":24,"bounds":{"left":0.5694444,"top":0.7211111,"width":0.08958333,"height":0.031111112},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Aneliya Angelova","depth":23,"bounds":{"left":0.6,"top":0.7922222,"width":0.056944445,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.7922222,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":15,"bounds":{"left":0.6,"top":0.7922222,"width":0.059722222,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.8155556,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.6888889,"top":0.79555553,"width":0.039583333,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"oki","depth":24,"bounds":{"left":0.5694444,"top":0.8411111,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":22,"bounds":{"left":0.6402778,"top":0.9166667,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"bounds":{"left":0.6,"top":0.97888887,"width":0.060416665,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reacted in","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.69027776,"top":0.9822222,"width":0.038194444,"height":0.012222222},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXCheckBox","text":"1 reaction, react with white check mark emoji","depth":24,"on_screen":false,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"1","depth":25,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Или си продължаваме същия тайм лайн","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Лукаш, оправи ли се с мейла?","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Home","depth":17,"bounds":{"left":0.7881944,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Messages","depth":17,"bounds":{"left":0.83194447,"top":0.12888889,"width":0.039583333,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Messages","depth":19,"bounds":{"left":0.83194447,"top":0.14,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"About","depth":17,"bounds":{"left":0.8909722,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Jump to date","depth":24,"bounds":{"left":0.8576389,"top":0.17555556,"width":0.05277778,"height":0.031111112},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Review Toast","depth":25,"bounds":{"left":0.8159722,"top":0.16,"width":0.061805554,"height":0.0011111111},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"APP","depth":25,"bounds":{"left":0.8826389,"top":0.16,"width":0.013194445,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":25,"bounds":{"left":0.8979167,"top":0.16,"width":0.0055555557,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Today at 4:38:01 PM","depth":25,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:38 PM","depth":26,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @nikolaybiaivanov","depth":26,"bounds":{"left":0.8159722,"top":0.16,"width":0.10763889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.060416665,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.8576389,"top":0.16,"width":0.013888889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"18 commits・29 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.16,"width":0.07430556,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.027083334,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20835","depth":28,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20835","depth":29,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Deployment notes:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.057638887,"height":0.027777778},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":16,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.057638887,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"• None","depth":28,"bounds":{"left":0.82708335,"top":0.20111111,"width":0.034027778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.26555556,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.26555556,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXLink","text":"Today at 4:40:26 PM","depth":26,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:40","depth":27,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":26,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":36,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":21,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446}}],"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.40111113,"width":0.068055555,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.40222222,"width":0.06527778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12106 Jy 20808 schedule reset on low queue","depth":28,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12106 Jy 20808 schedule reset on low queue","depth":29,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":42,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889}}],"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.875,"top":0.47555557,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.49888888,"width":0.06666667,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.5,"width":0.06388889,"height":0.02},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8284722,"top":0.5,"width":0.009027778,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.8375,"top":0.5,"width":0.05486111,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"33 commits・17 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":26,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.5822222,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20808","depth":28,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20808","depth":29,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"|","depth":28,"bounds":{"left":0.8979167,"top":0.5822222,"width":0.0055555557,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20928","depth":28,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20928","depth":29,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Notes:","depth":28,"bounds":{"left":0.82708335,"top":0.6311111,"width":0.030555556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Alter the selection algorithm:","depth":28,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":29,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"","depth":28,"bounds":{"left":0.87569445,"top":0.6888889,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Always take priority list FIRST.","depth":28,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":31,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08055556,"height":0.044444446}}],"role_description":"text"},{"role":"AXButton","text":"Show more","depth":27,"bounds":{"left":0.82708335,"top":0.7688889,"width":0.052083332,"height":0.024444444},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.83555555,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.83555555,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXCheckBox","text":"React with white_check_mark","depth":27,"bounds":{"left":0.8041667,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with eyes","depth":27,"bounds":{"left":0.8263889,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with raised_hands","depth":27,"bounds":{"left":0.8486111,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Add reaction…","depth":27,"bounds":{"left":0.87083334,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Reply in thread","depth":27,"bounds":{"left":0.89305556,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Forward message…","depth":27,"bounds":{"left":0.9152778,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"Save for later","depth":27,"bounds":{"left":0.9375,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"More actions","depth":27,"bounds":{"left":0.9597222,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"","depth":24,"bounds":{"left":0.7861111,"top":0.88,"width":0.19583334,"height":0.04222222},"on_screen":true,"value":"","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"}]...
|
1494787688392426485
|
-8571268630382073150
|
click
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
Reschedule
Mark as read
Clear
More actions
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-cb••HomeDMsActivityFilesLater..•MoreablAl chapter • 46 m left100% (C478•Wed 20 May 17:14:59EDQDescribe what you are looking forActivityAllDMs+C9 UnreadsNikolay Yan..DM12 minsаз сьм готов с моята част, мести ги вQА като видиш коментаритеJira Cloud20 minsApp_ Aneliya Angelova transitioned aStory you are assigned to from In QA...Review Toast35 mins::: AppPR review requested by @Vasil-JiminnyRemin...3:20 PMDMNikolay Yankov: ако искаш виж на пл...v Mark completeAneliya An...• DM12:19 PM 6okiYesterdayWea Stovan TanevYecterdavToastXHomeMessagesnotes:AboutToday ~• Nonejiminny/app994:40 PK review requested by @Vasil-JiminnyPR reviewrequested by@Vasil Vasilev#12106 Jy 20808schedule reset onlow queue by@Vasil Vasilev37 tles chane17 files changedJIRA: JY-20808 |JY-20928Notes:Alter the selectionalgorithm:Always takepriority list FIRST.Show morejiminny/appAdded by Toast for GitHubMessage Toast+Aa.*•...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64715
|
2266
|
7
|
2026-05-20T14:14:57.945141+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286497945_m1.jpg...
|
Slack
|
Toast (DM) - Jiminny Inc - 4 new items - Slack
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
3:20 PM
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"All, 4 unread","depth":19,"bounds":{"left":0.56041664,"top":0.13222222,"width":0.045138888,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXRadioButton","text":"DMs","depth":19,"bounds":{"left":0.60486114,"top":0.13222222,"width":0.03125,"height":0.036666665},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Add new tab","depth":20,"bounds":{"left":0.64375,"top":0.13222222,"width":0.011805556,"height":0.036666665},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Nikolay Yankov","depth":23,"bounds":{"left":0.6,"top":0.25444445,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.25444445,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.6,"top":0.25444445,"width":0.057638887,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.2777778,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12 mins","depth":23,"bounds":{"left":0.69722223,"top":0.25777778,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":24,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.5694444,"top":0.30333334,"width":0.17916666,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Jira Cloud","depth":23,"bounds":{"left":0.6,"top":0.39,"width":0.045833334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.41333333,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"20 mins","depth":23,"bounds":{"left":0.69722223,"top":0.39333335,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":24,"bounds":{"left":0.5847222,"top":0.43888888,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova transitioned a Story you are assigned to from","depth":24,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.5875,"top":0.43888888,"width":0.007638889,"height":0.02}},{"char_start":1,"char_count":61,"bounds":{"left":0.5694444,"top":0.43888888,"width":0.16736111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"In QA","depth":24,"bounds":{"left":0.7125,"top":0.46333334,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"⟶","depth":24,"bounds":{"left":0.5694444,"top":0.46333334,"width":0.18055555,"height":0.044444446},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ready To Merge","depth":24,"bounds":{"left":0.5833333,"top":0.48777777,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"…","depth":24,"bounds":{"left":0.65694445,"top":0.48777777,"width":0.008333334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Review Toast","depth":23,"bounds":{"left":0.6,"top":0.52555555,"width":0.059722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"App","depth":23,"bounds":{"left":0.6,"top":0.54888886,"width":0.016666668,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"35 mins","depth":23,"bounds":{"left":0.69722223,"top":0.5288889,"width":0.03125,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":24,"bounds":{"left":0.5694444,"top":0.5744445,"width":0.17847222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reminder","depth":23,"bounds":{"left":0.6,"top":0.63666666,"width":0.038194444,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.63666666,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":7,"bounds":{"left":0.6,"top":0.63666666,"width":0.036111113,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.66,"width":0.015972223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"3:20 PM","depth":23,"bounds":{"left":0.6944444,"top":0.64,"width":0.034027778,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov:","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.07430556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако искаш виж на планетата","depth":24,"bounds":{"left":0.5694444,"top":0.6855556,"width":0.18125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":25,"bounds":{"left":0.64305556,"top":0.6855556,"width":0.10486111,"height":0.02}}],"role_description":"text"},{"role":"AXButton","text":"Mark complete","depth":24,"bounds":{"left":0.5694444,"top":0.7211111,"width":0.08958333,"height":0.031111112},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Aneliya Angelova","depth":23,"bounds":{"left":0.6,"top":0.7922222,"width":0.056944445,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.6,"top":0.7922222,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":15,"bounds":{"left":0.6,"top":0.7922222,"width":0.059722222,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"bounds":{"left":0.6,"top":0.8155556,"width":0.015277778,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.6888889,"top":0.79555553,"width":0.039583333,"height":0.018888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"oki","depth":24,"bounds":{"left":0.5694444,"top":0.8411111,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":22,"bounds":{"left":0.6402778,"top":0.9166667,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"bounds":{"left":0.6,"top":0.97888887,"width":0.060416665,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Reacted in","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.69027776,"top":0.9822222,"width":0.038194444,"height":0.012222222},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXCheckBox","text":"1 reaction, react with white check mark emoji","depth":24,"on_screen":false,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"1","depth":25,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Или си продължаваме същия тайм лайн","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Лукаш, оправи ли се с мейла?","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"DM","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Home","depth":17,"bounds":{"left":0.7881944,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Messages","depth":17,"bounds":{"left":0.83194447,"top":0.12888889,"width":0.039583333,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Messages","depth":19,"bounds":{"left":0.83194447,"top":0.14,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"About","depth":17,"bounds":{"left":0.8909722,"top":0.12888889,"width":0.025,"height":0.04},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Jump to date","depth":24,"bounds":{"left":0.8576389,"top":0.17555556,"width":0.05277778,"height":0.031111112},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Review Toast","depth":25,"bounds":{"left":0.8159722,"top":0.16,"width":0.061805554,"height":0.0011111111},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"APP","depth":25,"bounds":{"left":0.8826389,"top":0.16,"width":0.013194445,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":25,"bounds":{"left":0.8979167,"top":0.16,"width":0.0055555557,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Today at 4:38:01 PM","depth":25,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:38 PM","depth":26,"bounds":{"left":0.90347224,"top":0.16,"width":0.03125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @nikolaybiaivanov","depth":26,"bounds":{"left":0.8159722,"top":0.16,"width":0.10763889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.060416665,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07777778,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12099 Jy 20835 mcp enable users to get a list of deals and their details","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.08125,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.8576389,"top":0.16,"width":0.013888889,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"nikolaybiaivanov","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"nikolaybiaivanov","depth":29,"bounds":{"left":0.82708335,"top":0.16,"width":0.07569444,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"18 commits・29 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.16,"width":0.07430556,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.027083334,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20835","depth":28,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20835","depth":29,"bounds":{"left":0.85347223,"top":0.16,"width":0.045138888,"height":0.0011111111},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Deployment notes:","depth":28,"bounds":{"left":0.82708335,"top":0.16,"width":0.057638887,"height":0.027777778},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.008333334,"height":0.02}},{"char_start":1,"char_count":16,"bounds":{"left":0.82708335,"top":0.14333333,"width":0.057638887,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"• None","depth":28,"bounds":{"left":0.82708335,"top":0.20111111,"width":0.034027778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.23666666,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.26555556,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.26555556,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.26555556,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXLink","text":"Today at 4:40:26 PM","depth":26,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"4:40","depth":27,"bounds":{"left":0.79375,"top":0.3011111,"width":0.016666668,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by @Vasil-Jiminny","depth":26,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":36,"bounds":{"left":0.8159722,"top":0.29777777,"width":0.14375,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"PR review requested by","depth":28,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":21,"bounds":{"left":0.82708335,"top":0.35333332,"width":0.060416665,"height":0.044444446}}],"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.40111113,"width":0.068055555,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.40222222,"width":0.06527778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"#12106 Jy 20808 schedule reset on low queue","depth":28,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"#12106 Jy 20808 schedule reset on low queue","depth":29,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":42,"bounds":{"left":0.82708335,"top":0.42666668,"width":0.08125,"height":0.06888889}}],"role_description":"text"},{"role":"AXStaticText","text":"by","depth":28,"bounds":{"left":0.875,"top":0.47555557,"width":0.013888889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"@Vasil Vasilev","depth":28,"bounds":{"left":0.82708335,"top":0.49888888,"width":0.06666667,"height":0.022222223},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"@Vasil Vasilev","depth":29,"bounds":{"left":0.8284722,"top":0.5,"width":0.06388889,"height":0.02},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.8284722,"top":0.5,"width":0.009027778,"height":0.02}},{"char_start":1,"char_count":13,"bounds":{"left":0.8375,"top":0.5,"width":0.05486111,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"33 commits・17 files changed","depth":29,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.00625,"height":0.02}},{"char_start":1,"char_count":26,"bounds":{"left":0.83819443,"top":0.5288889,"width":0.07430556,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"JIRA:","depth":28,"bounds":{"left":0.82708335,"top":0.5822222,"width":0.027083334,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20808","depth":28,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20808","depth":29,"bounds":{"left":0.85347223,"top":0.5822222,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"|","depth":28,"bounds":{"left":0.8979167,"top":0.5822222,"width":0.0055555557,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"JY-20928","depth":28,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"JY-20928","depth":29,"bounds":{"left":0.82708335,"top":0.6066667,"width":0.045138888,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Notes:","depth":28,"bounds":{"left":0.82708335,"top":0.6311111,"width":0.030555556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Alter the selection algorithm:","depth":28,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":29,"bounds":{"left":0.82708335,"top":0.66444445,"width":0.084027775,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"","depth":28,"bounds":{"left":0.87569445,"top":0.6888889,"width":0.0027777778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Always take priority list FIRST.","depth":28,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08125,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":31,"bounds":{"left":0.82708335,"top":0.7133333,"width":0.08055556,"height":0.044444446}}],"role_description":"text"},{"role":"AXButton","text":"Show more","depth":27,"bounds":{"left":0.82708335,"top":0.7688889,"width":0.052083332,"height":0.024444444},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"jiminny/app","depth":27,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"jiminny/app","depth":28,"bounds":{"left":0.82708335,"top":0.8066667,"width":0.047222223,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Added by","depth":27,"bounds":{"left":0.82708335,"top":0.83555555,"width":0.036805555,"height":0.016666668},"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Toast for GitHub","depth":27,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Toast for GitHub","depth":28,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.06111111,"height":0.016666668},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.86388886,"top":0.83555555,"width":0.004166667,"height":0.016666668}},{"char_start":1,"char_count":15,"bounds":{"left":0.8673611,"top":0.83555555,"width":0.057638887,"height":0.016666668}}],"role_description":"text"},{"role":"AXCheckBox","text":"React with white_check_mark","depth":27,"bounds":{"left":0.8041667,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with eyes","depth":27,"bounds":{"left":0.8263889,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"React with raised_hands","depth":27,"bounds":{"left":0.8486111,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Add reaction…","depth":27,"bounds":{"left":0.87083334,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Reply in thread","depth":27,"bounds":{"left":0.89305556,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Forward message…","depth":27,"bounds":{"left":0.9152778,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXCheckBox","text":"Save for later","depth":27,"bounds":{"left":0.9375,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"toggle button","subrole":"AXToggleButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"More actions","depth":27,"bounds":{"left":0.9597222,"top":0.26444444,"width":0.022222223,"height":0.035555556},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"","depth":24,"bounds":{"left":0.7861111,"top":0.88,"width":0.19583334,"height":0.04222222},"on_screen":true,"value":"","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"loading…","depth":11,"on_screen":false,"role_description":"text"}]...
|
-5201847798475362565
|
-8571267530878833982
|
click
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
All, 4 unread
DMs
Add new tab
Nikolay Yankov
DM
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Jira Cloud
App
20 mins
Aneliya Angelova transitioned a Story you are assigned to from
In QA
⟶
Ready To Merge
…
Review Toast
App
35 mins
PR review requested by @Vasil-Jiminny
Reminder
DM
3:20 PM
Nikolay Yankov:
ако искаш виж на планетата
Mark complete
Aneliya Angelova
DM
12:19 PM
oki
Yesterday
Stoyan Tanev
Reacted in
DM
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
1 reaction, react with white check mark emoji
1
Stoyan Tanev
DM
Yesterday
Или си продължаваме същия тайм лайн
James Graham
DM
Yesterday
2018
Stefka Stoyanova
DM
Yesterday
Лукаш, оправи ли се с мейла?
Galya Dimitrova
DM
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Home
Messages
Messages
About
Jump to date
Review Toast
APP
Today at 4:38:01 PM
4:38 PM
PR review requested by @nikolaybiaivanov
PR review requested by
nikolaybiaivanov
nikolaybiaivanov
#12099 Jy 20835 mcp enable users to get a list of deals and their details
#12099 Jy 20835 mcp enable users to get a list of deals and their details
by
nikolaybiaivanov
nikolaybiaivanov
18 commits・29 files changed
JIRA:
JY-20835
JY-20835
Deployment notes:
• None
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
Today at 4:40:26 PM
4:40
PR review requested by @Vasil-Jiminny
PR review requested by
@Vasil Vasilev
@Vasil Vasilev
#12106 Jy 20808 schedule reset on low queue
#12106 Jy 20808 schedule reset on low queue
by
@Vasil Vasilev
@Vasil Vasilev
33 commits・17 files changed
JIRA:
JY-20808
JY-20808
|
JY-20928
JY-20928
Notes:
Alter the selection algorithm:
Always take priority list FIRST.
Show more
jiminny/app
jiminny/app
Added by
Toast for GitHub
Toast for GitHub
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
loading…
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-cb••HomeDMsActivityFilesLater..•MoreablAl chapter • 46 m left100% (C478•Wed 20 May 17:14:57EDQDescribe what you are looking forActivityAllDMs+C9 UnreadsNikolay Yan..DM12 minsаз сьм готов с моята част, мести ги вQА като видиш коментаритеJira Cloud20 minsApp_ Aneliya Angelova transitioned aStory you are assigned to from In QA...Review Toast35 mins::: AppPR review requested by @Vasil-JiminnyRemin...3:20 PMDMNikolay Yankov: ако искаш виж на пл...v Mark completeAneliya An...• DM12:19 PM 6okiToastXHomeMessagesnotes:AboutToday ~• Nonejiminny/app4:40 PK review requested by @Vasil-JiminnyPR reviewrequested by@Vasil Vasilev#12106 Jy 20808schedule reset onlow queue by@Vasil Vasilev37 tles chane17 files changedJIRA: JY-20808 |JY-20928Notes:Alter the selectionalgorithm:Always takepriority list FIRST.Show morejiminny/appAdded by Toast for GitHubMessage Toast+Aa.*•YesterdayWea Stovan TanevYecterdav...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64714
|
2267
|
12
|
2026-05-20T14:14:55.169334+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286495169_m2.jpg...
|
Slack
|
Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Slack...
|
1
|
NULL
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.5152925,"top":1.0,"width":0.011968086,"height":-0.058260202},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXComboBox","text":"","depth":19,"bounds":{"left":0.54886967,"top":1.0,"width":0.08344415,"height":-0.09816444},"on_screen":true,"value":"","placeholder":"Find a DM","role_description":"combo box","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12 mins","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ако се наложи ще му пиша","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"}]...
|
-968733004847413901
|
-8544149050110800769
|
visual_change
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
rirero•• lPlatform Sprint 4 Q2 - Platfori-7 Service-Desk - Queues - Platforn(. JY-20676 delete AJ reports relateAllow owner's role to be selectedJY-20915 add alias for EU by Laky!Coverage on New Code - app in Ji# (JY-20915] Add environment-specl SevenShores|Hubspot|ExceptionslPipelines - jiminnvlapr(@ Jy 20808 schedule reset on low qu• Goodle geminiGemini•• XO JIMINNYconcurrenuy,they can both pass the if (! is_null(Redis::get(...))) check at the exact samemillisecond. Both will then execute set() and push duplicate metrics to Datadog.• The Fix: This should be replaced with a single atomic operation using Redis's NX (Not eXists)and EX (Expire) flags, or Laravel's built-in atomic cache locks:Redns::setsoaugelockName. true. "EX'. 60."NX.3. Infinite Loops and Caller ContractReturn Value Change: doUodateEntitiescnk( previouslv returned whether it should Iiterate again based on staleness or chunk size. It now always returns true (unless theselection is totally empty).The Bottleneck: If the code calling this method relies on a false return to pause, sleep, orgracefully terminate, changing this to an unconditional true could result in an aggressive"busy loop" that spikes CPU usage.4. Performance Overhead• Expensive Garbage Collection: Calling gc_collect_cycles(); on every single chunk can beCPU-intensive. While it is necessary since the restart logic was removed, it adds overhead tothe loop.Synchronous Metric Dispatch: If the Datadog: : increment or Datadog: :gauge methodsare making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standardStatsD), this will block the worker and significantly slow down the indexing process.5. Lack of Explicit Error Handling0 No try/catch Blocks: The script executes complex network operations ( LoadDocuments,Datadog logging, ElasticSearch updates/deletes) without visible exception handling in thissnippet. If a single document fails to load or the Elasticsearch cluster drops a connection, theunnanoled exceotion will crasn tne worker mio-oarch. oorenuiallv leavino tne oarch in aninconsictont ctatoAsk GeminilYour Jiminny chats aren't used to improve our models. Gemini is Al. It can make mistakes, so double check it. Your privacy & GeminiSummarize pageg For you@ RecentI# Starred8* AppsQ SpacesJiminny (New)+ .ll Plauorm leamIID Capture TeamaD Enterprise Stability I….II Processing TeamMl SE Kanbanl( Service-Desk= More spaces= FiltersB DashboardsC OperationsI2 Confluence:ã Teams2 Customise sidebar•y0 hil OAl chapter • 46 m left100% S• wea zu May 1/:14:02Q Search+ CreateAsk RovoSpaces / Jiminny (New)Platform Team P+® Summary& TimelineE BacklogID Active sprintsCalendar " Reports4 Testing Board# List & Forms &g Components% Development⅘ CodeO SecurityMore 7Q Search board00000Epic vType vQuick filtersvGroup: Queriesv Sorint (19 work items)READY FOR DEV 1IN DEV SUpgrade Bt librariesMayMAINTENANCEBacklog10000=Mcr> Enable users toget a list of calls andtheir detailsJIMINNY MCP CONNECTORIn Dev9e0=v JY-19958[ Jy-20833Improve Acuivity lypesuggestionsAUTO-DETECTED ACTVTYIn Dev2 %•=O JY-20410Add environment-snecific email domainsfor text relay to preve...In Dev=** JY-20915Complete sprintCODE REVIEW 3BLOCKEDQA2PO ACCEPTANCEDEPLOY 10MCP > Enable users toget a list of deals andtheir detailsJIMINNY MCP CONNECTORCode Review20 =[ JY-20835Upgrade to PHP 8.5PHP 8.5 UPGRADEIn QA1.5 0. 0000 =VJY-18091AJ Panorama for CallScoring in ODAUTOMATED AI SCORINGDeployed0.5 0 0000 =[ JY-20361Notify the user if aPanorama prompts isdeleted but is used in...AJ REPORTSCode ReviewMCP > Enable the AIto know details aboutthe userJIMINNY MCP CONNECTORReady for QA1.5 %6 00 =3 0=[ JY-20676… JY-20846[HubSpot] OptimiseCRM rematching ondelete hubsnot..PLATFORM STABILITYDeploved|1.5 /2 •=*7 JY-20725Notify the user if a SSis deleted but is usedin AJ ReportAJ REPORTSCode Review2.5 =D JY-20615[Deadline 25 May]Miarate denricatedGemini 3.1 Flash Lite….MAINTENANCEDeployed I1m•0 =N JY-20880Setup test coveragefor Prophet in SonarMAINTENANCEE JY-19951Upgrade Python andlibraries - May...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64713
|
2266
|
6
|
2026-05-20T14:14:54.235892+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286494235_m1.jpg...
|
Slack
|
Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Slack...
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages
Messages
Add canvas
Add canvas
More
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
Nikolay Yankov...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXComboBox","text":"","depth":19,"bounds":{"left":0.58194447,"top":0.13666667,"width":0.17430556,"height":0.018888889},"on_screen":true,"value":"","placeholder":"Find a DM","role_description":"combo box","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov","depth":22,"bounds":{"left":0.59097224,"top":0.19,"width":0.07083333,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12 mins","depth":23,"bounds":{"left":0.72847223,"top":0.19111112,"width":0.03125,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":23,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.16041666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.15972222,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova","depth":22,"bounds":{"left":0.59097224,"top":0.29555556,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.7201389,"top":0.29666665,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.32,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.32,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ако се наложи ще му пиша","depth":23,"bounds":{"left":0.61319447,"top":0.32,"width":0.13194445,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":22,"bounds":{"left":0.59097224,"top":0.37666667,"width":0.06111111,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.37777779,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.40111113,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.40111113,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":55,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":22,"bounds":{"left":0.59097224,"top":0.48222223,"width":0.068055555,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.48333332,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":23,"bounds":{"left":0.59097224,"top":0.50666666,"width":0.024305556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":22,"bounds":{"left":0.59097224,"top":0.56333333,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.5644444,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.5877778,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"в момента ползваме @txt.jiminny.com","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.5877778,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":34,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":22,"bounds":{"left":0.59097224,"top":0.66888887,"width":0.075,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.67,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":23,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.16597222,"height":0.06888889},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":73,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.16597222,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Vasil Vasilev","depth":22,"bounds":{"left":0.59097224,"top":0.77444446,"width":0.05625,"height":0.02},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.77444446,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":12,"bounds":{"left":0.59791666,"top":0.77444446,"width":0.049305554,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.77555555,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"благодаря","depth":23,"bounds":{"left":0.59097224,"top":0.79888886,"width":0.05,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tomov","depth":22,"bounds":{"left":0.59097224,"top":0.85555553,"width":0.06527778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Monday","depth":23,"bounds":{"left":0.7270833,"top":0.8566667,"width":0.03263889,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.88,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.88,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако се наложи ще го преместим","depth":23,"bounds":{"left":0.59097224,"top":0.88,"width":0.124305554,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.88,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":28,"bounds":{"left":0.59097224,"top":0.88,"width":0.124305554,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Todor Stamatov","depth":22,"bounds":{"left":0.59097224,"top":0.9611111,"width":0.072916664,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Thursday","depth":23,"bounds":{"left":0.72291666,"top":0.9622222,"width":0.036805555,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ахааа, то е failed и не се показва","depth":23,"bounds":{"left":0.59097224,"top":0.9855555,"width":0.15416667,"height":0.008888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Mario Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Ivanov","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще станат конфликти","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще го видя","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Petko Kashinski","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Huddle ?","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 29th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"да","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Ves","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 28th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXLink","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":23,"on_screen":false,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Messages","depth":18,"bounds":{"left":0.7826389,"top":0.12777779,"width":0.06458333,"height":0.04222222},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Messages","depth":20,"bounds":{"left":0.8020833,"top":0.14,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Add canvas","depth":19,"bounds":{"left":0.84930557,"top":0.12777779,"width":0.07152778,"height":0.04222222},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Add canvas","depth":21,"bounds":{"left":0.86875,"top":0.14,"width":0.046527777,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More","depth":19,"bounds":{"left":0.92291665,"top":0.12777779,"width":0.04236111,"height":0.04222222},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Add and Edit Channel Tabs","depth":18,"bounds":{"left":0.96458334,"top":0.12777779,"width":0.022916667,"height":0.04222222},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Canvas","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"List","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Folder","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXPopUpButton","text":"Jump to date","depth":23,"bounds":{"left":0.85,"top":0.16111112,"width":0.068055555,"height":0.0011111111},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Nikolay Yankov","depth":24,"bounds":{"left":0.8159722,"top":0.16111112,"width":0.072222225,"height":0.0011111111},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false}]...
|
-1393167100436403865
|
-7102775753051169697
|
visual_change
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
12 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages
Messages
Add canvas
Add canvas
More
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
Nikolay Yankov
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-eHomeDMsActivityFilesLaterMore+ablAl chapter • 46 m left100% C78•Wed 20 May 17:14:54EDDescribe what you are looking forDirect messages ~UnreadsQ Find a DMNikolay Yankov12 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayYou: Те би трябвало да влизат внова сделка новите activitiesJames Graham• 2018Stefka StoyanovaYou: в момента ползваме@txt.jiminny.comYesterdayGalya DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor Stamatov 0ThursdayNikolay…..MessagesAdd canvasMore ~ушнах за saved search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва да махнем recorder & voiceгледам че при мен май има самовалидацияте през FE ли са някьдедефинирани?Nikolay Yankov 2:37 PMЗащо, нали го има в условието?Lukas Kovalik 2:38 PMГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+Aa...
|
64707
|
NULL
|
NULL
|
NULL
|
|
64712
|
2267
|
11
|
2026-05-20T14:14:45.873708+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286485873_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
2770962430260122261
|
-1052937881026988068
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64711
|
2267
|
10
|
2026-05-20T14:14:42.870194+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286482870_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
652830244927763980
|
-1061874711537551396
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract...
|
64710
|
NULL
|
NULL
|
NULL
|
|
64710
|
2267
|
9
|
2026-05-20T14:14:41.161160+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286481161_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
6480823455465750553
|
-881731280493578340
|
click
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64709
|
2267
|
8
|
2026-05-20T14:14:39.853315+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286479853_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-9199961851185172050
|
-1079960028546893860
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on...
|
64708
|
NULL
|
NULL
|
NULL
|
|
64708
|
2267
|
7
|
2026-05-20T14:14:36.890502+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286476890_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
3899605034886564096
|
-1061875231230707748
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64707
|
2266
|
5
|
2026-05-20T14:14:33.586466+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286473586_m1.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
-4126034934648740211
|
-1052937881027004452
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64706
|
2267
|
6
|
2026-05-20T14:14:32.946744+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286472946_m2.jpg...
|
Firefox
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-5418029782153561499
|
-981939028105531191
|
click
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64704
|
2267
|
5
|
2026-05-20T14:14:32.132320+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286472132_m2.jpg...
|
Firefox
|
Service-Desk - Queues - Platform team - Service sp Service-Desk - Queues - Platform team - Service space - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/software/c/projects/JY/ jiminny.atlassian.net/jira/software/c/projects/JY/boards/37...
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Close tab
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.05905826,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.09177973,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
8419994565776980507
|
-881731280493578340
|
click
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Close tab
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Close tab
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt...
|
64702
|
NULL
|
NULL
|
NULL
|
|
64705
|
2266
|
4
|
2026-05-20T14:14:31.332339+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286471332_m1.jpg...
|
Firefox
|
Service-Desk - Queues - Platform team - Service sp Service-Desk - Queues - Platform team - Service space - Jira — Work...
|
1
|
jiminny.atlassian.net/jira/servicedesk/projects/SR jiminny.atlassian.net/jira/servicedesk/projects/SRD/queues/custom/37...
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DE SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obPhotos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obHomeDMsActivityFilesLater...More+ablAl chapter • 46 m left100% C78•Wed 20 May 17:14:30EDDescribe what you are looking forDirect messages ~UnreadsQ Find a DMNikolay Yankov11 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayYou: Те би трябвало да влизат внова сделка новите activitiesJames Graham• 2018YesterdayStefka StoyanovaYou: в момента ползваме@txt.jiminny.comYesterdayGalya DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor Stamatov 0ThursdayNikolay ...MessagesAdd canvasMore ~ушнах за saved search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва да махнем recorder & voiceгледам че при мен май има самовалидацияте през FE ли са някьдедефинирани?Nikolay Yankov 2:37 PMЗащо, нали го има в условието?Lukas Kovalik 2:38 PMГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+Aa...
|
NULL
|
7511111059804512645
|
NULL
|
click
|
ocr
|
NULL
|
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DE SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obPhotos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obHomeDMsActivityFilesLater...More+ablAl chapter • 46 m left100% C78•Wed 20 May 17:14:30EDDescribe what you are looking forDirect messages ~UnreadsQ Find a DMNikolay Yankov11 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayYou: Те би трябвало да влизат внова сделка новите activitiesJames Graham• 2018YesterdayStefka StoyanovaYou: в момента ползваме@txt.jiminny.comYesterdayGalya DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor Stamatov 0ThursdayNikolay ...MessagesAdd canvasMore ~ушнах за saved search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва да махнем recorder & voiceгледам че при мен май има самовалидацияте през FE ли са някьдедефинирани?Nikolay Yankov 2:37 PMЗащо, нали го има в условието?Lukas Kovalik 2:38 PMГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+Aa...
|
64703
|
NULL
|
NULL
|
NULL
|
|
64703
|
2266
|
3
|
2026-05-20T14:14:24.351124+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286464351_m1.jpg...
|
Slack
|
Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Slack...
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXComboBox","text":"","depth":19,"bounds":{"left":0.58194447,"top":0.13666667,"width":0.17430556,"height":0.018888889},"on_screen":true,"value":"","placeholder":"Find a DM","role_description":"combo box","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov","depth":22,"bounds":{"left":0.59097224,"top":0.19,"width":0.07083333,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"11 mins","depth":23,"bounds":{"left":0.72847223,"top":0.19111112,"width":0.03125,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":23,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.16041666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.15972222,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova","depth":22,"bounds":{"left":0.59097224,"top":0.29555556,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.7201389,"top":0.29666665,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.32,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.32,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ако се наложи ще му пиша","depth":23,"bounds":{"left":0.61319447,"top":0.32,"width":0.13194445,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":22,"bounds":{"left":0.59097224,"top":0.37666667,"width":0.06111111,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.37777779,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.40111113,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.40111113,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":55,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":22,"bounds":{"left":0.59097224,"top":0.48222223,"width":0.068055555,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.48333332,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":23,"bounds":{"left":0.59097224,"top":0.50666666,"width":0.024305556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":22,"bounds":{"left":0.59097224,"top":0.56333333,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.5644444,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.5877778,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"в момента ползваме @txt.jiminny.com","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.5877778,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":34,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446}}],"role_description":"text"}]...
|
4955274923377455770
|
-8545275491032920513
|
click
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴2Fixed 0 of 5690 files in 78.144 seconds, 60.00 MBmemory usedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:hotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obs-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-eHomeDMsActivityFilesLater...More+ablAl chapter • 46 m left100% C78•Wed 20 May 17:14:23EDDescribe what you are looking forDirect messages ~UnreadsQ Find a DMNikolay Yankov11 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayYou: Те би трябвало да влизат внова сделка новите activitiesJames Graham• 2018YesterdayStefka StoyanovaYou: в момента ползваме@txt.jiminny.comYesterdayGalya DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor Stamatov 0ThursdayNikolay ...MessagesAdd canvasMore ~ушнах за saved search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва да махнем recorder & voiceгледам че при мен май има самовалидацияте през FE ли са някьдедефинирани?Nikolay Yankov 2:37 PMЗащо, нали го има в условието?Lukas Kovalik 2:38 PMГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+AaTOD...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64702
|
2267
|
4
|
2026-05-20T14:14:22.464853+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286462464_m2.jpg...
|
Slack
|
Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Slack...
|
1
|
NULL
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages
Messages
Add canvas
Add canvas
More
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
Nikolay Yankov
Yesterday at 1:50:15 PM
1:50 PM
добре, иам коментари от claude
Lukas Kovalik...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.5152925,"top":1.0,"width":0.011968086,"height":-0.058260202},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"on_screen":true,"role_description":"text"},{"role":"AXComboBox","text":"","depth":19,"bounds":{"left":0.54886967,"top":1.0,"width":0.08344415,"height":-0.09816444},"on_screen":true,"value":"","placeholder":"Find a DM","role_description":"combo box","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"11 mins","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ако се наложи ще му пиша","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"в момента ползваме @txt.jiminny.com","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Vasil Vasilev","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"благодаря","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tomov","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Monday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако се наложи ще го преместим","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Todor Stamatov","depth":22,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Thursday","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ахааа, то е failed и не се показва","depth":23,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Mario Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Ivanov","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще станат конфликти","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще го видя","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Petko Kashinski","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Huddle ?","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 29th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"да","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Ves","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 28th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXLink","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":23,"on_screen":false,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Messages","depth":18,"bounds":{"left":0.6449468,"top":1.0,"width":0.030917553,"height":-0.09177971},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"Messages","depth":20,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Add canvas","depth":19,"bounds":{"left":0.6768617,"top":1.0,"width":0.034242023,"height":-0.09177971},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Add canvas","depth":21,"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More","depth":19,"bounds":{"left":0.71210104,"top":1.0,"width":0.020279255,"height":-0.09177971},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXPopUpButton","text":"Add and Edit Channel Tabs","depth":18,"bounds":{"left":0.73204786,"top":1.0,"width":0.010970744,"height":-0.09177971},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Canvas","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"List","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Folder","depth":18,"on_screen":false,"role_description":"text"},{"role":"AXPopUpButton","text":"Jump to date","depth":23,"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Nikolay Yankov","depth":24,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":24,"on_screen":true,"role_description":"text"},{"role":"AXLink","text":"Yesterday at 1:50:15 PM","depth":24,"on_screen":true,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"1:50 PM","depth":25,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"добре, иам коментари от claude","depth":25,"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Lukas Kovalik","depth":24,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":24,"on_screen":true,"role_description":"text"}]...
|
-5244569372701736659
|
-7066826140195646371
|
click
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages
Messages
Add canvas
Add canvas
More
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
Nikolay Yankov
Yesterday at 1:50:15 PM
1:50 PM
добре, иам коментари от claude
Lukas Kovalik
Platform Sprint 4 Q2 - Platform TeService-Desk - Queues - Platforn(. JY-20676 delete AJ reports rel XAllow owner's role to be selectedJY-20915 add alias for EU by Laky!Coverage on New Code - app in Ji# (JY-20915] Add environment-specl SevenShores|Hubspot|Exceptions|Pipelines - jiminnvlapr@ Jy 20808 schedule reset on low qu— New Tab• Goodle geminiGemini••• Xconcurrently,they can both pass the if (! is_null(Redis::get(...))) check at the exact samemillisecond. Both will then execute set() and push duplicate metrics to Datadog.• The Fix: This should be replaced with a single atomic operation using Redis's NX (Not eXists)and EX (Expire) flags, or Laravel's built-in atomic cache locks:Redis:: set($gaugeLockName, true, 'EX', 60,"NX.3. Infinite Loops and Caller ContractReturn Value Change: doUodateEntitieschik() previously returned whether it shoulditerate again based on staleness or chunk size. It now always returns true (unless theselection is totally empty).The Bottleneck: If the code calling this method relies on a false return to pause, sleep, orgracefully terminate, changing this to an unconditional true could result in an aggressive"busy loop" that spikes CPU usage.4. Performance Overhead• Expensive Garbage Collection: Calling gc_collect_cycles() ; on every single chunk can beCPU-intensive. While it is necessary since the restart logic was removed, it adds overhead tothe loop.Synchronous Metric Dispatch: If the Datadog: : increment or Datadog: :gauge methodsare making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standardStatsD), this will block the worker and significantly slow down the indexing process.5. Lack of Explicit Error Handling0 No try/catch Blocks: The script executes complex network operations ( LoadDocuments,Datadog logging, ElasticSearch updates/deletes) without visible exception handling in thissnippet. If a single document fails to load or the Elasticsearch cluster drops a connection, theunhandled exception will crash the worker mid-batch, potentially leaving the batch in aninconcictont ctateAsk GeminilYour Jiminny chats aren't used to improve our models. Gemini is Al. It can make mistakes, so double check it. Your privacy & GeminiSummarize pageC. 40 ln lAlchapter-46mleft O A 100% C/ &• Wed 20 May 17:14:2281 OpenJY-20676 delete AJ reports related objects #12098LakyLak wants to merge 20 commits into master from JY-20676-delete-report-related-objectsthe DiO.v• fetchActivitySearch silentlywrong has_reports - ActivityController.php:1083 now calls Ssearch->loadCount ( 'automatedReports') before transforming. &Still OpenMinor - Sthis->any() in testDeleteAskAnythingPromptAskAnvthinaPromotServiceTest.oho:367lsthis->askAnychingkepos1tory=>expects(sth1s=>any())|-willReturn(new Collection([l));This one instance of $this->any() was not fixed. Tracing AskAnythingPromptService::delete() for the owner path:FindSharedUsersAndGroupsByPromptId is called once inside recreatePromptsForEachRelation (which iterates thecollection — here emotv. so zero times) and once inside deletePromotIfNoRelations . So the expected count is.Sthis->onceo.Fix this →Minor — canExecute() is business loaic in a modelAutomatedReport: : canExecute() encapsulates a business rule (can't run without both a saved search and a prompt).Per project convention: "Never put business logic in models, use services instead." Not blocking, but worth a follow-upif not addressed here.SummaryThe PR is in very good shape. All critical and the vast majority of minor findings are resolved. Only one Sthis-›any()instance remains in the delete test, and the canExecute() convention issue persists. Neither is blocking — the PR isfunctionally correct and well-covered by tests.This hranch has not heen deniovadNo denlovmentsChanges approved1 approving review by reviewers with write access.y 1 approval ›& 1 pending review ›• Some checks haven't completed yet3 pending, 1 in progress, 1 expected, 15 successful checks...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64701
|
2266
|
2
|
2026-05-20T14:14:22.436974+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286462436_m1.jpg...
|
Slack
|
Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Nikolay Yankov (DM) - Jiminny Inc - 4 new items - Slack...
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages...
|
[{"role":"AXPopUpButton","text [{"role":"AXPopUpButton","text":"Switch workspaces… (Jiminny Inc) Has new messages","depth":14,"bounds":{"left":0.51180553,"top":0.08111111,"width":0.025,"height":0.04},"on_screen":true,"role_description":"pop-up button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXRadioButton","text":"Home","depth":14,"bounds":{"left":0.50625,"top":0.14,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Home","depth":16,"bounds":{"left":0.5138889,"top":0.19222222,"width":0.020833334,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"DMs","depth":14,"bounds":{"left":0.50625,"top":0.21555555,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false},{"role":"AXStaticText","text":"DMs","depth":16,"bounds":{"left":0.5159722,"top":0.26777777,"width":0.016666668,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"Activity","depth":14,"bounds":{"left":0.50625,"top":0.2911111,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Activity","depth":16,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.027083334,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51111114,"top":0.34333333,"width":0.0055555557,"height":0.015555556}},{"char_start":1,"char_count":7,"bounds":{"left":0.5159722,"top":0.34333333,"width":0.022222223,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Files","depth":14,"bounds":{"left":0.50625,"top":0.36666667,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Files","depth":16,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.015972223,"height":0.015555556},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.51666665,"top":0.4188889,"width":0.004166667,"height":0.015555556}},{"char_start":1,"char_count":4,"bounds":{"left":0.5208333,"top":0.4188889,"width":0.011805556,"height":0.015555556}}],"role_description":"text"},{"role":"AXRadioButton","text":"Later","depth":14,"bounds":{"left":0.50625,"top":0.4422222,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Later","depth":16,"bounds":{"left":0.5152778,"top":0.49444443,"width":0.018055556,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXRadioButton","text":"More…","depth":14,"bounds":{"left":0.50625,"top":0.5177778,"width":0.036111113,"height":0.075555556},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"More","depth":16,"bounds":{"left":0.5152778,"top":0.57,"width":0.01875,"height":0.015555556},"on_screen":true,"role_description":"text"},{"role":"AXComboBox","text":"","depth":19,"bounds":{"left":0.58194447,"top":0.13666667,"width":0.17430556,"height":0.018888889},"on_screen":true,"value":"","placeholder":"Find a DM","role_description":"combo box","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"","depth":20,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Yankov","depth":22,"bounds":{"left":0.59097224,"top":0.19,"width":0.07083333,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"11 mins","depth":23,"bounds":{"left":0.72847223,"top":0.19111112,"width":0.03125,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"аз съм готов с моята част, мести ги в QA като видиш коментарите","depth":23,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.16041666,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":62,"bounds":{"left":0.59097224,"top":0.21444444,"width":0.15972222,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova","depth":22,"bounds":{"left":0.59097224,"top":0.29555556,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12:19 PM","depth":23,"bounds":{"left":0.7201389,"top":0.29666665,"width":0.039583333,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.32,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.32,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Ако се наложи ще му пиша","depth":23,"bounds":{"left":0.61319447,"top":0.32,"width":0.13194445,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tanev","depth":22,"bounds":{"left":0.59097224,"top":0.37666667,"width":0.06111111,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.37777779,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.40111113,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Те би трябвало да влизат в нова сделка новите activities","depth":23,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.40111113,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":55,"bounds":{"left":0.59097224,"top":0.40111113,"width":0.15486111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"James Graham","depth":22,"bounds":{"left":0.59097224,"top":0.48222223,"width":0.068055555,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.48333332,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"2018","depth":23,"bounds":{"left":0.59097224,"top":0.50666666,"width":0.024305556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stefka Stoyanova","depth":22,"bounds":{"left":0.59097224,"top":0.56333333,"width":0.08055556,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.5644444,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.5877778,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"в момента ползваме @txt.jiminny.com","depth":23,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.5877778,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":34,"bounds":{"left":0.59097224,"top":0.5877778,"width":0.12361111,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Galya Dimitrova","depth":22,"bounds":{"left":0.59097224,"top":0.66888887,"width":0.075,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.67,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е","depth":23,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.16597222,"height":0.06888889},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":73,"bounds":{"left":0.59097224,"top":0.6933333,"width":0.16597222,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Vasil Vasilev","depth":22,"bounds":{"left":0.59097224,"top":0.77444446,"width":0.05625,"height":0.02},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.59097224,"top":0.77444446,"width":0.0069444445,"height":0.02}},{"char_start":1,"char_count":12,"bounds":{"left":0.59791666,"top":0.77444446,"width":0.049305554,"height":0.02}}],"role_description":"text"},{"role":"AXStaticText","text":"Yesterday","depth":23,"bounds":{"left":0.72152776,"top":0.77555555,"width":0.038194444,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"благодаря","depth":23,"bounds":{"left":0.59097224,"top":0.79888886,"width":0.05,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Stoyan Tomov","depth":22,"bounds":{"left":0.59097224,"top":0.85555553,"width":0.06527778,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Monday","depth":23,"bounds":{"left":0.7270833,"top":0.8566667,"width":0.03263889,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"You:","depth":23,"bounds":{"left":0.59097224,"top":0.88,"width":0.02013889,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"","depth":23,"bounds":{"left":0.61041665,"top":0.88,"width":0.0034722222,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ако се наложи ще го преместим","depth":23,"bounds":{"left":0.59097224,"top":0.88,"width":0.124305554,"height":0.044444446},"on_screen":true,"lines":[{"char_start":0,"char_count":1,"bounds":{"left":0.61319447,"top":0.88,"width":0.0055555557,"height":0.02}},{"char_start":1,"char_count":28,"bounds":{"left":0.59097224,"top":0.88,"width":0.124305554,"height":0.044444446}}],"role_description":"text"},{"role":"AXStaticText","text":"Todor Stamatov","depth":22,"bounds":{"left":0.59097224,"top":0.9611111,"width":0.072916664,"height":0.02},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Thursday","depth":23,"bounds":{"left":0.72291666,"top":0.9622222,"width":0.036805555,"height":0.017777778},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"ахааа, то е failed и не се показва","depth":23,"bounds":{"left":0.59097224,"top":0.9855555,"width":0.15416667,"height":0.008888889},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"Mario Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Nikolay Ivanov","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Wednesday","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще станат конфликти","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"ще го видя","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Petko Kashinski","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"May 12th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Huddle ?","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 29th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"да","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"Ves","depth":22,"on_screen":false,"role_description":"text"},{"role":"AXStaticText","text":"April 28th","depth":23,"on_screen":false,"role_description":"text"},{"role":"AXLink","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":23,"on_screen":false,"role_description":"link","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"https://github.com/jiminny/extension-app/tree/master/public","depth":24,"on_screen":false,"role_description":"text"},{"role":"AXRadioButton","text":"Messages","depth":18,"bounds":{"left":0.7826389,"top":0.12777779,"width":0.06458333,"height":0.04222222},"on_screen":true,"role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true,"is_expanded":false}]...
|
4832141879552659410
|
-7102995672556594091
|
click
|
hybrid
|
NULL
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Nikolay Yankov
11 mins
аз съм готов с моята част, мести ги в QA като видиш коментарите
Aneliya Angelova
12:19 PM
You:
Ако се наложи ще му пиша
Stoyan Tanev
Yesterday
You:
Те би трябвало да влизат в нова сделка новите activities
James Graham
Yesterday
2018
Stefka Stoyanova
Yesterday
You:
в момента ползваме @txt.jiminny.com
Galya Dimitrova
Yesterday
едно е ок, няма нужда да ги делим. Той като влезе на едит ще си види кое е
Vasil Vasilev
Yesterday
благодаря
Stoyan Tomov
Monday
You:
ако се наложи ще го преместим
Todor Stamatov
Thursday
ахааа, то е failed и не се показва
Mario Georgiev
Wednesday
тенк ю вери мъч, махнах му crm seat-a и има достъп, ще го имам напредвид вече че явно бие такава грешка когато фрии акаунта expir-ne или нещо подобно и няма особен workaround
Nikolay Ivanov
Wednesday
ще станат конфликти
Steliyan Georgiev
May 12th
ще го видя
Petko Kashinski
May 12th
Huddle ?
Aneliya Angelova, Nikolay Yankov, Steliyan Georgiev
April 29th
да
Ves
April 28th
https://github.com/jiminny/extension-app/tree/master/public
https://github.com/jiminny/extension-app/tree/master/public
Messages
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER₴81DEV (-zsh)₴82Fixed 0 of 5690 files in 78.144 seconds, 60.00MBmemoryusedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] (JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obhotos-3-001\(2\)/talon-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-ob]s-3-001\(2\)/vin.png/Users/lukas/Downloads/Photos-3-001(2)/vin.HEIC/Users/lukas/Downloads/Photos-3-001(2)/vin.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob:Photos-3-001\(2\)/dr-lic-f.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/app (JY-20676-delete-report-related-ob]Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obis-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob]s-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-ob/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obHomeDMsActivityFilesLaterMore+(aholAl chapter - 46 m left100% <28• Wed 20 May 17:14:22Describe what you are looking forDirect messages ~UnreadsQ Find a DMNikolay Yankov11 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayo You: Те би трябвало да влизат внова сделка новите activitiesJames Graham• 2018YesterdayStefka StoyanovaYou: в момента ползваме@txt.jiminny.comYesterdayGalya DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor Stamatov 0ThursdayNikolay ...MessagesAdd canvasMore ~ушнах за saved search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва да махнем recorder & voiceгледам че при мен май има самовалидацияте през FE ли са някьдедефинирани?Nikolay Yankov 2:37 PMЗащо, нали го има в условието?Lukas Kovalik 2:38 PMГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+AaTOD...
|
64698
|
NULL
|
NULL
|
NULL
|
|
64700
|
2267
|
3
|
2026-05-20T14:14:11.359131+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286451359_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests
(
31
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality
Security and quality
Insights
Insights
Settings
Settings
JY-20676 delete AJ reports related objects #12098 Edit title
JY-20676 delete AJ reports related objects
#
12098
Edit title
Checks pending
Checks pending
Code
Code
Open
LakyLak
LakyLak
wants to merge 20 commits into
master
master
from
JY-20676-delete-report-related-objects
JY-20676-delete-report-related-objects
Copy head branch name to clipboard
Lines changed: 1698 additions & 1471 deletions
Conversation (9)
Conversation
(
9
)...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Upload & tools","depth":20,"bounds":{"left":0.12732713,"top":0.8567438,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Microphone","depth":20,"bounds":{"left":0.32413563,"top":0.86153233,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Send message","depth":20,"bounds":{"left":0.33211437,"top":0.8631285,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":false,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.","depth":17,"bounds":{"left":0.102726065,"top":0.9173983,"width":0.21775267,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Your privacy & Gemini Opens in a new window","depth":17,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your privacy & Gemini","depth":18,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Opens in a new window","depth":19,"bounds":{"left":0.079288565,"top":0.91660017,"width":0.04720745,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Summarize page","depth":7,"bounds":{"left":0.08494016,"top":0.95730245,"width":0.053523935,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Summarize page","depth":9,"bounds":{"left":0.09059176,"top":0.96249,"width":0.042220745,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Skip to content","depth":6,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Skip to content","depth":7,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Open menu","depth":10,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"Homepage (g then d)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"jiminny","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"jiminny","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"app","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"app","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Search or jump to…","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Type","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"/","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to search","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Chat with Copilot","depth":10,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXMenuButton","text":"Open Copilot…","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXMenuButton","text":"Create new...","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"All issues(g then i)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All pull requests","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All repositories","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"You have unread notifications(g then n)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open user navigation menu","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Repository navigation","depth":9,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Repository navigation","depth":10,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Code","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Code","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Pull requests (31)","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pull requests","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"31","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":")","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Agents","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Agents","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Actions","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Actions","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Wiki","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Wiki","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Security and quality","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Security and quality","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Insights","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Insights","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Settings","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Settings","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"JY-20676 delete AJ reports related objects #12098 Edit title","depth":13,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"#","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"12098","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Edit title","depth":14,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Checks pending","depth":13,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Checks pending","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXMenuButton","text":"Code","depth":13,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Code","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Open","depth":13,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"LakyLak","depth":15,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"LakyLak","depth":16,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"wants to merge 20 commits into","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"master","depth":15,"on_screen":false,"role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"master","depth":16,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"from","depth":16,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"JY-20676-delete-report-related-objects","depth":16,"on_screen":false,"role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20676-delete-report-related-objects","depth":17,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy head branch name to clipboard","depth":16,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Lines changed: 1698 additions & 1471 deletions","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Conversation (9)","depth":16,"on_screen":false,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"Conversation","depth":17,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(","depth":18,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"9","depth":18,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":")","depth":18,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-7087746738428694851
|
-1052787213578504420
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests
(
31
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality
Security and quality
Insights
Insights
Settings
Settings
JY-20676 delete AJ reports related objects #12098 Edit title
JY-20676 delete AJ reports related objects
#
12098
Edit title
Checks pending
Checks pending
Code
Code
Open
LakyLak
LakyLak
wants to merge 20 commits into
master
master
from
JY-20676-delete-report-related-objects
JY-20676-delete-report-related-objects
Copy head branch name to clipboard
Lines changed: 1698 additions & 1471 deletions
Conversation (9)
Conversation
(
9
)...
|
64699
|
NULL
|
NULL
|
NULL
|
|
64699
|
2267
|
2
|
2026-05-20T14:14:08.343878+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286448343_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"}]...
|
988148017449334202
|
-1079959444431538212
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64698
|
2266
|
1
|
2026-05-20T14:14:05.465634+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286445465_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
4344092198535999235
|
-1026409384166083752
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64697
|
2267
|
1
|
2026-05-20T14:13:53.227582+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286433227_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests
(
31
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality
Security and quality
Insights
Insights
Settings
Settings
JY-20676 delete AJ reports related objects #12098 Edit title
JY-20676 delete AJ reports related objects
#
12098
Edit title
Checks pending
Checks pending
Code
Code
Open...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Upload & tools","depth":20,"bounds":{"left":0.12732713,"top":0.8567438,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Microphone","depth":20,"bounds":{"left":0.32413563,"top":0.86153233,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Send message","depth":20,"bounds":{"left":0.33211437,"top":0.8631285,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":false,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.","depth":17,"bounds":{"left":0.102726065,"top":0.9173983,"width":0.21775267,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Your privacy & Gemini Opens in a new window","depth":17,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your privacy & Gemini","depth":18,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Opens in a new window","depth":19,"bounds":{"left":0.079288565,"top":0.91660017,"width":0.04720745,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Summarize page","depth":7,"bounds":{"left":0.08494016,"top":0.95730245,"width":0.053523935,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Summarize page","depth":9,"bounds":{"left":0.09059176,"top":0.96249,"width":0.042220745,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Skip to content","depth":6,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Skip to content","depth":7,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Open menu","depth":10,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"Homepage (g then d)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"jiminny","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"jiminny","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"app","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"app","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Search or jump to…","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Type","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"/","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to search","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Chat with Copilot","depth":10,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXMenuButton","text":"Open Copilot…","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXMenuButton","text":"Create new...","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"All issues(g then i)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All pull requests","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All repositories","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"You have unread notifications(g then n)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open user navigation menu","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Repository navigation","depth":9,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Repository navigation","depth":10,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Code","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Code","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Pull requests (31)","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pull requests","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"31","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":")","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Agents","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Agents","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Actions","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Actions","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Wiki","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Wiki","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Security and quality","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Security and quality","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Insights","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Insights","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Settings","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Settings","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"JY-20676 delete AJ reports related objects #12098 Edit title","depth":13,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"#","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"12098","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Edit title","depth":14,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Checks pending","depth":13,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Checks pending","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXMenuButton","text":"Code","depth":13,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Code","depth":15,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Open","depth":13,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-6189565768846781858
|
-1070810957936625892
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests
(
31
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality
Security and quality
Insights
Insights
Settings
Settings
JY-20676 delete AJ reports related objects #12098 Edit title
JY-20676 delete AJ reports related objects
#
12098
Edit title
Checks pending
Checks pending
Code
Code
Open...
|
64696
|
NULL
|
NULL
|
NULL
|
|
64696
|
2267
|
0
|
2026-05-20T14:13:42.341552+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286422341_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Upload & tools","depth":20,"bounds":{"left":0.12732713,"top":0.8567438,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Microphone","depth":20,"bounds":{"left":0.32413563,"top":0.86153233,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Send message","depth":20,"bounds":{"left":0.33211437,"top":0.8631285,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":false,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.","depth":17,"bounds":{"left":0.102726065,"top":0.9173983,"width":0.21775267,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Your privacy & Gemini Opens in a new window","depth":17,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your privacy & Gemini","depth":18,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Opens in a new window","depth":19,"bounds":{"left":0.079288565,"top":0.91660017,"width":0.04720745,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Summarize page","depth":7,"bounds":{"left":0.08494016,"top":0.95730245,"width":0.053523935,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Summarize page","depth":9,"bounds":{"left":0.09059176,"top":0.96249,"width":0.042220745,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Skip to content","depth":6,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Skip to content","depth":7,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Open menu","depth":10,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"Homepage (g then d)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"jiminny","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"jiminny","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"app","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"app","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Search or jump to…","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Type","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"/","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to search","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Chat with Copilot","depth":10,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXMenuButton","text":"Open Copilot…","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXMenuButton","text":"Create new...","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"All issues(g then i)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All pull requests","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All repositories","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"You have unread notifications(g then n)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open user navigation menu","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Repository navigation","depth":9,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Repository navigation","depth":10,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Code","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Code","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Pull requests (31)","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pull requests","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
5720481319462978766
|
-1052796009671460964
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)
Pull requests...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64695
|
2266
|
0
|
2026-05-20T14:13:35.015697+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286415015_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
7512898529376068722
|
-1052937881027004452
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response...
|
64693
|
NULL
|
NULL
|
NULL
|
|
64694
|
NULL
|
0
|
2026-05-20T14:13:11.536299+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286391536_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
2165610740144057478
|
-917759493397006436
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter...
|
64692
|
NULL
|
NULL
|
NULL
|
|
64693
|
NULL
|
0
|
2026-05-20T14:13:04.588194+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286384588_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-3789728784885741914
|
-1079889110047098916
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64692
|
2265
|
67
|
2026-05-20T14:12:41.030032+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286361030_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
1062163830408800351
|
-1070952279540680932
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64691
|
2264
|
29
|
2026-05-20T14:12:34.196963+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286354196_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-3304065254231117893
|
-1061874711537616996
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the...
|
64687
|
NULL
|
NULL
|
NULL
|
|
64690
|
2265
|
66
|
2026-05-20T14:12:10.485329+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286330485_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-5042768006313656563
|
-1061874711537616932
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
64689
|
NULL
|
NULL
|
NULL
|
|
64689
|
2265
|
65
|
2026-05-20T14:12:07.482162+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286327482_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Upload & tools","depth":20,"bounds":{"left":0.12732713,"top":0.8567438,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Microphone","depth":20,"bounds":{"left":0.32413563,"top":0.86153233,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Send message","depth":20,"bounds":{"left":0.33211437,"top":0.8631285,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":false,"is_focused":false,"is_selected":false}]...
|
-4892789105667151914
|
-1052937846671460580
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64688
|
2265
|
64
|
2026-05-20T14:12:04.466848+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286324466_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-4682791562070316493
|
-1061874711537551460
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock...
|
64686
|
NULL
|
NULL
|
NULL
|
|
64687
|
2264
|
28
|
2026-05-20T14:12:03.762450+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286323762_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-5042768006313656563
|
-1061874711537616932
|
idle
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64686
|
2265
|
63
|
2026-05-20T14:12:01.389410+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286321389_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
4566640402469731839
|
-1079889110047033380
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64685
|
2265
|
62
|
2026-05-20T14:11:58.374328+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286318374_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
8150348838347613474
|
-1061874681472845924
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response...
|
64684
|
NULL
|
NULL
|
NULL
|
|
64684
|
2265
|
61
|
2026-05-20T14:11:55.344293+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286315344_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-412904074452865308
|
-1079959478791211044
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64683
|
2265
|
60
|
2026-05-20T14:11:52.321423+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286312321_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-8621523292286619302
|
-1061874711537616932
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()...
|
64682
|
NULL
|
NULL
|
NULL
|
|
64682
|
2265
|
59
|
2026-05-20T14:11:49.335587+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286309335_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-7347303376054114365
|
-1061874711537616932
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64681
|
2265
|
58
|
2026-05-20T14:11:46.304657+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286306304_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
1526423920121232508
|
-1061874711537551460
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same...
|
64680
|
NULL
|
NULL
|
NULL
|
|
64680
|
2265
|
57
|
2026-05-20T14:11:43.286996+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286303286_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
1062163830408800351
|
-1070952279540680932
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64679
|
2265
|
56
|
2026-05-20T14:11:37.269434+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286297269_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
8963481789372004064
|
-1070952279536486436
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export...
|
64678
|
NULL
|
NULL
|
NULL
|
|
64678
|
2265
|
55
|
2026-05-20T14:11:34.220983+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286294220_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage....
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
7056788041877329843
|
-1079889110047033380
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage....
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64676
|
2264
|
27
|
2026-05-20T14:11:33.372418+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286293372_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:...
|
[{"role":"AXStaticText","text& [{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"github.com","depth":4,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-1588784419904871481
|
-1079889110047033380
|
click
|
accessibility
|
NULL
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:...
|
64675
|
NULL
|
NULL
|
NULL
|
|
64677
|
2265
|
54
|
2026-05-20T14:11:33.353602+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286293353_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (...
|
[{"role":"AXStaticText","text& [{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.08361037,"top":0.1245012,"width":0.08427527,"height":0.021548284},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"github.com","depth":4,"bounds":{"left":0.08361037,"top":0.14604948,"width":0.019614361,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
238207153713282688
|
-1079889075687295012
|
click
|
accessibility
|
NULL
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (...
|
64674
|
NULL
|
NULL
|
NULL
|
|
64675
|
2264
|
26
|
2026-05-20T14:11:31.875520+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286291875_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Allow owner's role to be selected when setting Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Close tab
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
[{"role":"AXStaticText","text& [{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"github.com","depth":4,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"}]...
|
-3501240900674981015
|
-1061874711537616936
|
visual_change
|
accessibility
|
NULL
|
Allow owner's role to be selected when setting Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
github.com
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Close tab
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64674
|
2265
|
53
|
2026-05-20T14:11:31.187750+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286291187_m2.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Close tab
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"bounds":{"left":0.0,"top":0.0518755,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.06304868,"width":0.10106383,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":4,"bounds":{"left":0.0,"top":0.08459697,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Service-Desk - Queues - Platform team - Service space - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.09577015,"width":0.10721409,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.11731844,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":true},{"role":"AXStaticText","text":"JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.12849163,"width":0.15791224,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.1245012,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.15003991,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.16121309,"width":0.1796875,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Close tab","depth":5,"bounds":{"left":0.06732048,"top":0.15722266,"width":0.007978723,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXRadioButton","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.18276137,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.19393456,"width":0.12699468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":4,"bounds":{"left":0.0,"top":0.21548285,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Coverage on New Code - app in Jiminny SonarQube Cloud","depth":5,"bounds":{"left":0.013297873,"top":0.22665602,"width":0.10139628,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":4,"bounds":{"left":0.0,"top":0.2482043,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira","depth":5,"bounds":{"left":0.013297873,"top":0.25937748,"width":0.18068483,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":4,"bounds":{"left":0.0,"top":0.28092578,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"SevenShores\\Hubspot\\Exceptions\\BadRequest: Client error: `POST https://api.hubapi.com/crm/v3/objects/contact/search` resulted in a `429 Too Many Requests` response: {\"status\":\"error\",\"message\":\"You have reached your secondly limit.\",\"errorType\":\"RATE_LIMIT","depth":5,"bounds":{"left":0.013297873,"top":0.29209897,"width":0.4644282,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Pipelines - jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.31364724,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Pipelines - jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.32482043,"width":0.039228722,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXRadioButton","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":4,"bounds":{"left":0.0,"top":0.3463687,"width":0.07962101,"height":0.032721467},"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Jy 20808 schedule reset on low queue by Vasil-Jiminny · Pull Request #12106 · jiminny/app","depth":5,"bounds":{"left":0.013297873,"top":0.3575419,"width":0.15824468,"height":0.010774142},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Tab","depth":4,"bounds":{"left":0.0028257978,"top":0.38068634,"width":0.07413564,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Customize sidebar","depth":6,"bounds":{"left":0.0028257978,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Close Google Gemini (⌃X)","depth":6,"bounds":{"left":0.013796543,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Tabs from other devices","depth":6,"bounds":{"left":0.024933511,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open history (⇧⌘H)","depth":6,"bounds":{"left":0.036070477,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Open bookmarks (⌘B)","depth":6,"bounds":{"left":0.04720745,"top":0.97007185,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"AI Chat settings","depth":7,"bounds":{"left":0.35854387,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Close","depth":7,"bounds":{"left":0.37051198,"top":0.055067837,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Main menu","depth":13,"bounds":{"left":0.08228058,"top":0.09736632,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"New chat","depth":12,"bounds":{"left":0.09823803,"top":0.10215483,"width":0.028091755,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Gemini","depth":15,"bounds":{"left":0.1008976,"top":0.10654429,"width":0.02144282,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"New Chat","depth":13,"bounds":{"left":0.35854387,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open menu for conversation actions.","depth":13,"bounds":{"left":0.37051198,"top":0.10215483,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Conversation with Gemini","depth":15,"bounds":{"left":0.079288565,"top":0.14445332,"width":0.0003324468,"height":0.0007980846},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Conversation with Gemini","depth":16,"bounds":{"left":0.079288565,"top":0.14684756,"width":0.1200133,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"You said I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\\LaravelDatadogHelper\\Datadog;7use ChaseConey\\LaravelDatadogHelper\\Datadog;8use Illuminate\\Support\\Facades\\Log;8use Illuminate\\Support\\Facades\\Log;9+use Illuminate\\Support\\Facades\\Redis;9use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\Client as ElasticSearchClient;10use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ElasticSearch\\ElasticSearchMappings;11use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\DeleteDocumentsAction;12use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\LoadDocumentsAction;13use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14use Jiminny\\Component\\ES\\Processor\\Actions\\UpsertDocumentsAction;14-use Jiminny\\Component\\ES\\Processor\\BatchStatusInterface;15+use Jiminny\\Component\\ES\\Processor\\BatchStatusManager;15use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SelectionList;16use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\DTOs\\SimpleCollection;17use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\TargetEntitiesSelector;18use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\ExtractIdsTrait;19use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\SelectEntityListTrait;20use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21use Jiminny\\Component\\ES\\Processor\\Traits\\UpdateTargetTrait;21-use Jiminny\\Component\\ES\\Processor\\Traits\\WorkerIdTrait;22use Jiminny\\Utils\\StringUtil;22use Jiminny\\Utils\\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.","depth":23,"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Expand","depth":21,"on_screen":false,"role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a concise summary of the changes made in the code selection:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Refactoring and Dependencies","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Refactoring and Dependencies","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Replaced Interface with Implementation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Changed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusInterface","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"BatchStatusManager","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"in the imports and constructor dependency injection.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Redis Support:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Imported","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Illuminate\\Support\\Facades\\Redis","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to handle a newly introduced memory-logging lock mechanism.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Traits:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"WorkerIdTrait","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", along with the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"worker","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"field tracking in the final logs.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Loop and Worker Lifecycle Adjustments","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Loop and Worker Lifecycle Adjustments","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Worker Auto-Restart:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Deleted the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property, the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"constant, and the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"shouldIterateAgain()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method. The script no longer tracks execution iterations or triggers a self-managed worker restart.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Simplified Chunk Method Return:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Modified","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to always return","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Priority Handling for Throttled Entities","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Priority Handling for Throttled Entities","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Stale Record Elevation:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Updated the logic in","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"rescheduleThrottledEntities()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to check if a throttled entity group is classified as \"stale\". If it is, the entities are rescheduled using","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"QueuePriorityEnum::NORMAL","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"; otherwise, they keep their respective queue priority.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"Metrics, Logging, and Environment Handling","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Metrics, Logging, and Environment Handling","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added Environment Tracking:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Introduced a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$environment","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"property (defaulting to","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"'local'","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") and a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"setEnvironment()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method to dynamically assign it.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Memory Monitoring via Datadog Gauge:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added a","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method that submits peak memory allocation to Datadog as a gauge metric (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"jiminny.reindex-memory-usage","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis Lock for Metrics Throttling:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Implemented a Redis-backed lock (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"%s-process-manager-lock","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") with a 60-second expiration inside","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to prevent over-reporting metrics.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Log Payload Update:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Added total processed items (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$updatedDocuments + $deletedDocuments","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":") to the final information log.","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":23,"on_screen":false,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":23,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":22,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"You said are there any issues or potential bottlenecks","depth":21,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"You said","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"are there any issues or potential bottlenecks","depth":23,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Copy prompt","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Edit","depth":20,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Gemini said","depth":20,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Gemini said","depth":21,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Here is a breakdown of the potential issues to look out for:","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"1. Memory Leak Risk (OOM Crashes)","depth":23,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"1. Memory Leak Risk (OOM Crashes)","depth":24,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Removed Restart Logic:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The most significant change is the removal of the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$counter","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"RESTART_AFTER","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":").","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Dependency Leaks:","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If any underlying service (like the","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ElasticSearchClient","depth":27,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", ORM, or Datadog helper) retains references to processed entities,","depth":26,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles()","depth":27,"bounds":{"left":0.26811835,"top":0.0,"width":0.056848403,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23138298,"height":0.05546688},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Hardcoded Limits:","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.047041222,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The code still includes","depth":26,"bounds":{"left":0.17154256,"top":0.0,"width":0.05651596,"height":0.01715882},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"ini_set('memory_limit', '512M');","depth":27,"bounds":{"left":0.23005319,"top":0.0,"width":0.095744684,"height":0.015163607},"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":". Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.","depth":26,"bounds":{"left":0.12450133,"top":0.0,"width":0.23055187,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"2. Race Condition in Redis Lock","depth":23,"bounds":{"left":0.111369684,"top":0.06823623,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"2. Race Condition in Redis Lock","depth":24,"bounds":{"left":0.111369684,"top":0.06783719,"width":0.09125665,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Non-Atomic Operations:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.062333778,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The","depth":26,"bounds":{"left":0.18683511,"top":0.09497207,"width":0.011635638,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"logMemoryUsage","depth":27,"bounds":{"left":0.20046543,"top":0.096568234,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"method uses three distinct steps to handle the lock:","depth":26,"bounds":{"left":0.12450133,"top":0.09497207,"width":0.22722739,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"get()","depth":27,"bounds":{"left":0.14860372,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":",","depth":26,"bounds":{"left":0.16555852,"top":0.11412609,"width":0.002493351,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.17004654,"top":0.11572227,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", and","depth":26,"bounds":{"left":0.18700133,"top":0.11412609,"width":0.013131649,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"expire()","depth":27,"bounds":{"left":0.20212767,"top":0.11572227,"width":0.023936171,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.22805852,"top":0.11412609,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If multiple workers process the same","depth":26,"bounds":{"left":0.16456117,"top":0.14285715,"width":0.09275266,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"$this->updateTarget","depth":27,"bounds":{"left":0.25930852,"top":0.14445332,"width":0.056848403,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"concurrently, they can both pass the","depth":26,"bounds":{"left":0.12450133,"top":0.14285715,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"if (! is_null(Redis::get(...)))","depth":27,"bounds":{"left":0.18351063,"top":0.16360734,"width":0.09275266,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"check at the exact same millisecond. Both will then execute","depth":26,"bounds":{"left":0.12450133,"top":0.16201118,"width":0.21575798,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"set()","depth":27,"bounds":{"left":0.2122673,"top":0.18276137,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"and push duplicate metrics to Datadog.","depth":26,"bounds":{"left":0.22922207,"top":0.1811652,"width":0.09823803,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Fix:","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"This should be replaced with a single atomic operation using Redis's","depth":26,"bounds":{"left":0.14378324,"top":0.20989625,"width":0.16821809,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"NX","depth":27,"bounds":{"left":0.31399602,"top":0.21149242,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Not eXists) and","depth":26,"bounds":{"left":0.12450133,"top":0.20989625,"width":0.22855718,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"EX","depth":27,"bounds":{"left":0.13696809,"top":0.23064645,"width":0.005984043,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(Expire) flags, or Laravel’s built-in atomic cache locks:","depth":26,"bounds":{"left":0.14494681,"top":0.22905028,"width":0.13264628,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Redis::set($gaugeLockName, true, 'EX', 60, 'NX')","depth":27,"bounds":{"left":0.12649602,"top":0.24980047,"width":0.14361702,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":".","depth":26,"bounds":{"left":0.27210772,"top":0.2482043,"width":0.0013297872,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"3. Infinite Loops and Caller Contract","depth":23,"bounds":{"left":0.111369684,"top":0.2980846,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"3. Infinite Loops and Caller Contract","depth":24,"bounds":{"left":0.111369684,"top":0.29768556,"width":0.10571808,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Return Value Change:","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.055352394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"doUpdateEntitiesChunk()","depth":27,"bounds":{"left":0.18301196,"top":0.3264166,"width":0.06881649,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"previously returned whether it should iterate again based on staleness or chunk size. It now always returns","depth":26,"bounds":{"left":0.12450133,"top":0.32482043,"width":0.2237367,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.29438165,"top":0.34557062,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"(unless the selection is totally empty).","depth":26,"bounds":{"left":0.12450133,"top":0.34397447,"width":0.2130984,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The Bottleneck:","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.040059842,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the code calling this method relies on a","depth":26,"bounds":{"left":0.16456117,"top":0.39185953,"width":0.10305851,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"false","depth":27,"bounds":{"left":0.26961437,"top":0.3934557,"width":0.014960106,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"return to pause, sleep, or gracefully terminate, changing this to an unconditional","depth":26,"bounds":{"left":0.12450133,"top":0.39185953,"width":0.22589761,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"true","depth":27,"bounds":{"left":0.26163563,"top":0.41260973,"width":0.011968086,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"could result in an aggressive \"busy loop\" that spikes CPU usage.","depth":26,"bounds":{"left":0.12450133,"top":0.41101357,"width":0.22290559,"height":0.03631285},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"4. Performance Overhead","depth":23,"bounds":{"left":0.111369684,"top":0.48004788,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"4. Performance Overhead","depth":24,"bounds":{"left":0.111369684,"top":0.47964883,"width":0.076130316,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Expensive Garbage Collection:","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.078457445,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Calling","depth":26,"bounds":{"left":0.20295878,"top":0.5067837,"width":0.019281914,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"gc_collect_cycles();","depth":27,"bounds":{"left":0.22423537,"top":0.5083799,"width":0.059840426,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"on","depth":26,"bounds":{"left":0.28607047,"top":0.5067837,"width":0.00880984,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"every single chunk","depth":26,"bounds":{"left":0.29488033,"top":0.5067837,"width":0.045545213,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.","depth":26,"bounds":{"left":0.12450133,"top":0.5067837,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Synchronous Metric Dispatch:","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.07712766,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"If the","depth":26,"bounds":{"left":0.20162898,"top":0.5738228,"width":0.015292553,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::increment","depth":27,"bounds":{"left":0.21891622,"top":0.575419,"width":0.053856384,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"or","depth":26,"bounds":{"left":0.27476728,"top":0.5738228,"width":0.0078125,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Datadog::gauge","depth":27,"bounds":{"left":0.28457448,"top":0.575419,"width":0.041888297,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.","depth":26,"bounds":{"left":0.12450133,"top":0.5738228,"width":0.23420878,"height":0.05546688},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXHeading","text":"5. Lack of Explicit Error Handling","depth":23,"bounds":{"left":0.111369684,"top":0.66201115,"width":0.24734043,"height":0.01915403},"on_screen":true,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"5. Lack of Explicit Error Handling","depth":24,"bounds":{"left":0.111369684,"top":0.66161215,"width":0.094913565,"height":0.020351157},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"No","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.008477394,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"try/catch","depth":27,"bounds":{"left":0.1349734,"top":0.6903432,"width":0.026928192,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Blocks:","depth":26,"bounds":{"left":0.16389628,"top":0.688747,"width":0.019614361,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"The script executes complex network operations (","depth":26,"bounds":{"left":0.18351063,"top":0.688747,"width":0.12367021,"height":0.01715882},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"loadDocuments","depth":27,"bounds":{"left":0.30917552,"top":0.6903432,"width":0.038896278,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":", Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.","depth":26,"bounds":{"left":0.12450133,"top":0.688747,"width":0.22888963,"height":0.09377494},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXCheckBox","text":"Good response","depth":22,"bounds":{"left":0.1087101,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXCheckBox","text":"Bad response","depth":22,"bounds":{"left":0.124667555,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"toggle button","subrole":"AXToggle","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Redo","depth":22,"bounds":{"left":0.140625,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Share & export","depth":21,"bounds":{"left":0.15658244,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Copy","depth":22,"bounds":{"left":0.17253989,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Show more options","depth":21,"bounds":{"left":0.18849733,"top":0.79768556,"width":0.011968086,"height":0.028731046},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXTextArea","text":"Ask Gemini","depth":21,"bounds":{"left":0.14594415,"top":0.86632085,"width":0.17021276,"height":0.01915403},"on_screen":true,"value":"Ask Gemini","help_text":"","role_description":"text entry area","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Ask Gemini","depth":22,"bounds":{"left":0.14594415,"top":0.8667199,"width":0.030418882,"height":0.018355945},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Upload & tools","depth":20,"bounds":{"left":0.12732713,"top":0.8567438,"width":0.015957447,"height":0.03830806},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Microphone","depth":20,"bounds":{"left":0.32413563,"top":0.86153233,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Send message","depth":20,"bounds":{"left":0.33211437,"top":0.8631285,"width":0.010638298,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":false,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.","depth":17,"bounds":{"left":0.102726065,"top":0.9173983,"width":0.21775267,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Your privacy & Gemini Opens in a new window","depth":17,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Your privacy & Gemini","depth":18,"bounds":{"left":0.32047874,"top":0.9173983,"width":0.043882977,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Opens in a new window","depth":19,"bounds":{"left":0.079288565,"top":0.91660017,"width":0.04720745,"height":0.013567438},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Summarize page","depth":7,"bounds":{"left":0.08494016,"top":0.95730245,"width":0.053523935,"height":0.025538707},"on_screen":true,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Summarize page","depth":9,"bounds":{"left":0.09059176,"top":0.96249,"width":0.042220745,"height":0.015163607},"on_screen":true,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Skip to content","depth":6,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Skip to content","depth":7,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Open menu","depth":10,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"Homepage (g then d)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"jiminny","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"jiminny","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"app","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"app","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXButton","text":"Search or jump to…","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Type","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"/","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXStaticText","text":"to search","depth":12,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Chat with Copilot","depth":10,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXMenuButton","text":"Open Copilot…","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXMenuButton","text":"Create new...","depth":9,"on_screen":false,"help_text":"","role_description":"menu button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXLink","text":"All issues(g then i)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All pull requests","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"All repositories","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXLink","text":"You have unread notifications(g then n)","depth":9,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXButton","text":"Open user navigation menu","depth":9,"on_screen":false,"help_text":"","role_description":"button","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXHeading","text":"Repository navigation","depth":9,"on_screen":false,"help_text":"","role_description":"heading","subrole":"AXUnknown"},{"role":"AXStaticText","text":"Repository navigation","depth":10,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Code","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false},{"role":"AXStaticText","text":"Code","depth":14,"on_screen":false,"help_text":"","role_description":"text","subrole":"AXUnknown"},{"role":"AXLink","text":"Pull requests (31)","depth":12,"on_screen":false,"help_text":"","role_description":"link","subrole":"AXUnknown","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
5088377340139915549
|
-1052797109183088740
|
visual_change
|
accessibility
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
Service-Desk - Queues - Platform team - Service space - Jira
Service-Desk - Queues - Platform team - Service space - Jira
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app
Close tab
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Allow owner's role to be selected when setting up a trial by LakyLak · Pull Request #12092 · jiminny/app
Close tab
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
JY-20915 add alias for EU by LakyLak · Pull Request #12105 · jiminny/app
Coverage on New Code - app in Jiminny SonarQube Cloud
Coverage on New Code - app in Jiminny SonarQube Cloud
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
[JY-20915] Add environment-specific email domains for text relay to prevent duplicate processing - Jira
SevenShores\Hubspot\Exceptions\BadRequest: Client error: `POST [URL_WITH_CREDENTIALS] -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected. Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
You said
I’m on page “<tabTitle>Jy 20808 schedule reset on low queue by Vasil-Jimi</tabTitle>” with “<selection>@@ -6,38 +6,35 @@667use ChaseConey\LaravelDatadogHelper\Datadog;7use ChaseConey\LaravelDatadogHelper\Datadog;8use Illuminate\Support\Facades\Log;8use Illuminate\Support\Facades\Log;9+use Illuminate\Support\Facades\Redis;9use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\Client as ElasticSearchClient;10use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ElasticSearch\ElasticSearchMappings;11use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\DeleteDocumentsAction;12use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\LoadDocumentsAction;13use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14use Jiminny\Component\ES\Processor\Actions\UpsertDocumentsAction;14-use Jiminny\Component\ES\Processor\BatchStatusInterface;15+use Jiminny\Component\ES\Processor\BatchStatusManager;15use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SelectionList;16use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\DTOs\SimpleCollection;17use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\TargetEntitiesSelector;18use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\ExtractIdsTrait;19use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\SelectEntityListTrait;20use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21use Jiminny\Component\ES\Processor\Traits\UpdateTargetTrait;21-use Jiminny\Component\ES\Processor\Traits\WorkerIdTrait;22use Jiminny\Utils\StringUtil;22use Jiminny\Utils\StringUtil;232324class UpdateProcessManager24class UpdateProcessManager25{25{26use ExtractIdsTrait;26use ExtractIdsTrait;27use SelectEntityListTrait;27use SelectEntityListTrait;28use UpdateTargetTrait;28use UpdateTargetTrait;29-use WorkerIdTrait;302931-/**30+private const string GAUGE_LOCK_KEY = '%s-process-manager-lock';32- * Restart the script every X iterations, to allow the OS to collect any unused memory31+33- */32+private string $environment = 'local';34-private const int RESTART_AFTER = 20; // Iterations35-private int $counter = 0;363337public function __construct(34public function __construct(38private readonly ElasticSearchClient $esClient,35private readonly ElasticSearchClient $esClient,39private readonly TargetEntitiesSelector $entitySelector,36private readonly TargetEntitiesSelector $entitySelector,40-private readonly BatchStatusInterface $batchStatusManager,37+private readonly BatchStatusManager $batchStatusManager,41private readonly DeleteDocumentsAction $deleteDocumentsAction,38private readonly DeleteDocumentsAction $deleteDocumentsAction,42private readonly UpsertDocumentsAction $upsertDocumentsAction,39private readonly UpsertDocumentsAction $upsertDocumentsAction,43private readonly LoadDocumentsAction $loadDocumentsAction,40private readonly LoadDocumentsAction $loadDocumentsAction,@@ -49,6 +46,11 @@ public function __construct(49ini_set('memory_limit', '512M');46ini_set('memory_limit', '512M');50 }47 }514849+public function setEnvironment(string $environment): void50+ {51+$this->environment = $environment;52+ }53+52public function bootstrap(): void54public function bootstrap(): void53 {55 {54// Prepare the batch status manager56// Prepare the batch status manager@@ -73,21 +75,15 @@ public function bootstrap(): void737574public function doUpdateEntitiesChunk(): bool76public function doUpdateEntitiesChunk(): bool75 {77 {76-$batchName = $this->batchStatusManager->generateName();77$startTime = microtime(true);78$startTime = microtime(true);79+$batchName = $this->batchStatusManager->generateName();788079/** Select Entity IDs for processing */81/** Select Entity IDs for processing */80$selectionList = $this->entitySelector->select();82$selectionList = $this->entitySelector->select();81if ($selectionList->isEmpty()) {83if ($selectionList->isEmpty()) {82return false;84return false;83 }85 }848685-/**86- * A stale chunk will always be smaller than a regular chunk.87- * That doesn't mean there aren't any other scheduled entities for processing88- */89-$shouldIterateAgain = $this->shouldIterateAgain($selectionList);90-91/** Mark the batch as scheduled for work, then load the actual documents */87/** Mark the batch as scheduled for work, then load the actual documents */92$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);88$this->batchStatusManager->scheduleWork($selectionList->getAllIds(), $batchName);93$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());89$documentLoad = $this->loadDocumentsAction->loadDocuments($this->updateTarget, $selectionList->getAllIds());@@ -103,42 +99,23 @@ public function doUpdateEntitiesChunk(): bool103 unset($selectionList);99 unset($selectionList);104gc_collect_cycles();100gc_collect_cycles();105101106-/**102+/** Write stats and logs */107- * Write stats and logs108- */109 Datadog::increment(103 Datadog::increment(110 stats: 'jiminny.es-documents-synced',104 stats: 'jiminny.es-documents-synced',111 sampleRate: 1,105 sampleRate: 1,112 value: $updatedDocuments + $deletedDocuments106 value: $updatedDocuments + $deletedDocuments113 );107 );114108109+$this->logMemoryUsage();110+115 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [111 Log::info('[ EsUpdateProcessManager ] Finished updating entities in ES', [116-'worker' => $this->workerId,117'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),112'peak_memory' => StringUtil::filesizeToHuman(memory_get_peak_usage(true)),118'elapsed_seconds' => round(microtime(true) - $startTime, 2),113'elapsed_seconds' => round(microtime(true) - $startTime, 2),119'update_target' => $this->updateTarget,114'update_target' => $this->updateTarget,120-'should_iterate_again' => $shouldIterateAgain,115+'processed' => $updatedDocuments + $deletedDocuments,121 ]);116 ]);122117123-$this->counter++;118+return true;124-125-return $shouldIterateAgain;126- }127-128-private function shouldIterateAgain(SelectionList $selectionList): bool129- {130-if ($selectionList->isStaleChunkSelected() || $selectionList->isFullChunk()) {131-/**132- * Restart the worker to allow the OS to collect any unused memory133- */134-if ($this->counter >= self::RESTART_AFTER) {135-return false;136- }137-138-return true;139- }140-141-return false;142 }119 }143120144private function executeDelete(SimpleCollection $documentsToDelete): int121private function executeDelete(SimpleCollection $documentsToDelete): int@@ -162,13 +139,41 @@ private function rescheduleThrottledEntities(SelectionList $selectionList): void162$groups = $selectionList->getEntityListGroups();139$groups = $selectionList->getEntityListGroups();163foreach ($groups as $eachGroup) {140foreach ($groups as $eachGroup) {164$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);141$groupEntityIds = $selectionList->getEntityListItemsByGroup($eachGroup);142+$groupPriority = QueuePriorityEnum::tryFromString($eachGroup);143+165if (! empty($groupEntityIds)) {144if (! empty($groupEntityIds)) {145+/**146+ * Throttled stale records are rescheduled as normal priority147+ * all other records are rescheduled to their respective queue148+ */166 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(149 AsyncUpdateElasticSearch::bulkMarkEntitiesForUpdate(167 entityType: $this->updateTarget,150 entityType: $this->updateTarget,168 entityIds: $groupEntityIds,151 entityIds: $groupEntityIds,169- priority: QueuePriorityEnum::tryFromString($eachGroup)152+ priority: $groupPriority->isStale() ? QueuePriorityEnum::NORMAL : $groupPriority,170 );153 );171 }154 }172 }155 }173 }156 }157+158+private function logMemoryUsage(): void159+ {160+$gaugeLockName = sprintf(self::GAUGE_LOCK_KEY, $this->updateTarget);161+if (! is_null(Redis::get($gaugeLockName))) {162+return;163+ }164+165+ Redis::set($gaugeLockName, true);166+ Redis::expire($gaugeLockName, 60);167+168+// Add memory usage of the process as well169+ Datadog::gauge(170+ stat: 'jiminny.reindex-memory-usage',171+ value: memory_get_peak_usage(true),172+ sampleRate: 1,173+ tags: [174+'environment' => $this->environment,175+'type' => $this->updateTarget,176+ ]177+ );178+ }174}179}</selection>” selected.
Please summarize the selection using precise and concise language. Use headers and bulleted lists in the summary, to make it scannable. Maintain the meaning and factual accuracy.
Expand
Copy prompt
Gemini said
Gemini said
Here is a concise summary of the changes made in the code selection:
Refactoring and Dependencies
Refactoring and Dependencies
Replaced Interface with Implementation:
Changed
BatchStatusInterface
to
BatchStatusManager
in the imports and constructor dependency injection.
Added Redis Support:
Imported
Illuminate\Support\Facades\Redis
to handle a newly introduced memory-logging lock mechanism.
Removed Traits:
Removed
WorkerIdTrait
, along with the
worker
field tracking in the final logs.
Loop and Worker Lifecycle Adjustments
Loop and Worker Lifecycle Adjustments
Removed Worker Auto-Restart:
Deleted the
$counter
property, the
RESTART_AFTER
constant, and the
shouldIterateAgain()
method. The script no longer tracks execution iterations or triggers a self-managed worker restart.
Simplified Chunk Method Return:
Modified
doUpdateEntitiesChunk()
to always return
true
upon completion, rather than determining whether to iterate again based on chunk fullness or staleness.
Priority Handling for Throttled Entities
Priority Handling for Throttled Entities
Stale Record Elevation:
Updated the logic in
rescheduleThrottledEntities()
to check if a throttled entity group is classified as "stale". If it is, the entities are rescheduled using
QueuePriorityEnum::NORMAL
; otherwise, they keep their respective queue priority.
Metrics, Logging, and Environment Handling
Metrics, Logging, and Environment Handling
Added Environment Tracking:
Introduced a
$environment
property (defaulting to
'local'
) and a
setEnvironment()
method to dynamically assign it.
Memory Monitoring via Datadog Gauge:
Added a
logMemoryUsage()
method that submits peak memory allocation to Datadog as a gauge metric (
jiminny.reindex-memory-usage
).
Redis Lock for Metrics Throttling:
Implemented a Redis-backed lock (
%s-process-manager-lock
) with a 60-second expiration inside
logMemoryUsage()
to prevent over-reporting metrics.
Log Payload Update:
Added total processed items (
$updatedDocuments + $deletedDocuments
) to the final information log.
Good response
Bad response
Share & export
Copy
Show more options
You said are there any issues or potential bottlenecks
You said
are there any issues or potential bottlenecks
Copy prompt
Edit
Gemini said
Gemini said
Based on the code selection provided, there are several potential issues, bottlenecks, and architectural risks—primarily revolving around memory management, race conditions, and error handling.
Here is a breakdown of the potential issues to look out for:
1. Memory Leak Risk (OOM Crashes)
1. Memory Leak Risk (OOM Crashes)
Removed Restart Logic:
The most significant change is the removal of the
$counter
and
RESTART_AFTER
logic. Previously, the worker restarted itself to let the OS reclaim memory. Now, it relies entirely on PHP's internal garbage collection (
gc_collect_cycles()
).
Dependency Leaks:
If any underlying service (like the
ElasticSearchClient
, ORM, or Datadog helper) retains references to processed entities,
gc_collect_cycles()
will not be able to free them. The worker will eventually hit the memory limit and crash with a fatal Out Of Memory (OOM) error.
Hardcoded Limits:
The code still includes
ini_set('memory_limit', '512M');
. Hardcoding environment constraints inside application logic makes it difficult to scale or tune workers dynamically via infrastructure configurations.
2. Race Condition in Redis Lock
2. Race Condition in Redis Lock
Non-Atomic Operations:
The
logMemoryUsage
method uses three distinct steps to handle the lock:
get()
,
set()
, and
expire()
.
The Bottleneck:
If multiple workers process the same
$this->updateTarget
concurrently, they can both pass the
if (! is_null(Redis::get(...)))
check at the exact same millisecond. Both will then execute
set()
and push duplicate metrics to Datadog.
The Fix:
This should be replaced with a single atomic operation using Redis's
NX
(Not eXists) and
EX
(Expire) flags, or Laravel’s built-in atomic cache locks:
Redis::set($gaugeLockName, true, 'EX', 60, 'NX')
.
3. Infinite Loops and Caller Contract
3. Infinite Loops and Caller Contract
Return Value Change:
doUpdateEntitiesChunk()
previously returned whether it should iterate again based on staleness or chunk size. It now always returns
true
(unless the selection is totally empty).
The Bottleneck:
If the code calling this method relies on a
false
return to pause, sleep, or gracefully terminate, changing this to an unconditional
true
could result in an aggressive "busy loop" that spikes CPU usage.
4. Performance Overhead
4. Performance Overhead
Expensive Garbage Collection:
Calling
gc_collect_cycles();
on
every single chunk
can be CPU-intensive. While it is necessary since the restart logic was removed, it adds overhead to the loop.
Synchronous Metric Dispatch:
If the
Datadog::increment
or
Datadog::gauge
methods are making synchronous API calls over HTTP or TCP (rather than non-blocking UDP like standard StatsD), this will block the worker and significantly slow down the indexing process.
5. Lack of Explicit Error Handling
5. Lack of Explicit Error Handling
No
try/catch
Blocks:
The script executes complex network operations (
loadDocuments
, Datadog logging, ElasticSearch updates/deletes) without visible exception handling in this snippet. If a single document fails to load or the Elasticsearch cluster drops a connection, the unhandled exception will crash the worker mid-batch, potentially leaving the batch in an inconsistent state.
Good response
Bad response
Redo
Share & export
Copy
Show more options
Ask Gemini
Ask Gemini
Upload & tools
Microphone
Send message
Your Jiminny chats aren’t used to improve our models. Gemini is AI. It can make mistakes, so double check it.
Your privacy & Gemini Opens in a new window
Your privacy & Gemini
Opens in a new window
Summarize page
Summarize page
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (31)...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64673
|
2264
|
25
|
2026-05-20T14:11:28.843329+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286288843_m1.jpg...
|
Firefox
|
JY-20676 delete AJ reports related objects by Laky JY-20676 delete AJ reports related objects by LakyLak · Pull Request #12098 · jiminny/app — Work...
|
1
|
github.com/jiminny/app/pull/12098
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira...
|
[{"role":"AXRadioButton","text [{"role":"AXRadioButton","text":"Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira","depth":4,"on_screen":true,"help_text":"","role_description":"tab","subrole":"AXTabButton","is_enabled":true,"is_focused":false,"is_selected":false}]...
|
8852295099697676704
|
7230047640003280981
|
visual_change
|
hybrid
|
NULL
|
Platform Sprint 4 Q2 - Platform Team - Scrum Board Platform Sprint 4 Q2 - Platform Team - Scrum Board - Jira
SlackFileEditViewGoHistoryWindowHelpAPPDOCKER• ₴1DEV (-zsh)₴2Fixedof 5690 files in 78.144 seconds,60.00MBmemoryusedWhat's next:Try Docker Debug for seamless, persistentdebugging tools in any containeror image →Learn more at [URL_WITH_CREDENTIALS] ~/jiminny/app(JY-20676-delete-report-related-obihotos-3-001\(2\)/talon-b.png/Usere /ulene/Daualaade/Dhatne 2 A0100s/balanh UCTGlukd5-3-/USEPSlukcPhot/USESlackLukas-Photos-3-001\(2\)/dr-lic-b.png/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/dr-lic-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2)/gtp.pngWarning: /Users/lukas/Downloads/Photos-3-001(2)/gtp.HEIC not a valid file - skippinglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20676-delete-report-related-obs-3-001\(2\)/gtp.png/Users/lukas/Downloads/Photos-3-001(2)/gtp.heif/Users/lukas/Downloads/Photos-3-001(2)/gtp.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny~/jiminny/app (JY-20676-delete-report-related-ob:/Photos-3-001\(2\)/talon-m-f.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-f.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/Jiminny/appJY-20676-delete-report-related-ob]/Photos-3-001\(2\)/talon-m-b.png/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.HEIC/Users/lukas/Downloads/Photos-3-001(2)/talon-m-b.pnglukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app(JY-20676-delete-report-related-obHomeDMsActivity• 0(ahol•Al chapter • 49 m left100% <28•Wed 20 May 17:11:28Describe what you are looking forDirect messages ~UnreadsQFind a DMNikolay Yankov8 minsаз сьм готов с моята част, местиги в QА като видиш коментаритеAneliya Angelova12:19 PMYou: Ако се наложи ще му пишаStoyan TanevYesterdayNikolay ...MessagesAdd canvasMore vпушнах за saven search в сьщиябранчToday ~+Saved for later • Due 2 hours agoако искаш виж на планетатаNikolay Yankov 1:01 PMЛукас, има коментари от ClaudeLukas Kovalik 2:35 PMНики при таска за owner rolesтрябва ла махнем recorder & [EMAIL] DimitrovaYesterdayедно е ок, няма нужда да гиделим. Той като влезе на едит щ...Vasil Vasilev XблагодаряYesterdayStoyan TomovYou: ако се наложи ще гопреместимMondayTodor StamatovThursdayNГаля иска да се махнеNikolay Yankov 2:38 PMАха, добре ,ще го махна да я нямав списькаNikolay Yankov 3:25 PMмахнах гоNewNikolay Yankov 5:03 PMаз сьм готов с моята част, местиги в QА като видиш коментаритеMessage Nikolay Yankov+Aa...
|
64671
|
NULL
|
NULL
|
NULL
|
|
64672
|
2265
|
52
|
2026-05-20T14:11:24.888418+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286284888_m2.jpg...
|
PhpStorm
|
faVsco.js – console [PROD]
|
1
|
NULL
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification...
|
[{"role":"AXButton","text" [{"role":"AXButton","text":"Project: faVsco.js, menu","depth":5,"bounds":{"left":0.025930852,"top":0.019952115,"width":0.03856383,"height":0.025538707},"on_screen":true,"help_text":"~/jiminny/app","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"#12105 on JY-20915-add-domain-specific-emai…relay, menu","depth":5,"bounds":{"left":0.064494684,"top":0.019952115,"width":0.124667555,"height":0.025538707},"on_screen":true,"help_text":"Pull request #12105 exists for current branch JY-20915-add-domain-specific-emai…relay","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Start Listening for PHP Debug Connections","depth":5,"bounds":{"left":0.8450798,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"SmsReceivedTest","depth":6,"bounds":{"left":0.86037236,"top":0.019952115,"width":0.05518617,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Run 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9155585,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Debug 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9268617,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"More Actions","depth":6,"bounds":{"left":0.9381649,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"JetBrains AI","depth":5,"bounds":{"left":0.96609044,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Search Everywhere","depth":5,"bounds":{"left":0.9773936,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"IDE and Project Settings","depth":5,"bounds":{"left":0.9886968,"top":0.019952115,"width":0.011303186,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false}]...
|
15234730114371202
|
-8960939456853734463
|
visual_change
|
hybrid
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
PhostormVIewINavicarecodeWindowFV faVsco.js#12105 on JY-20915-add-domain-specificroledey©) WebhookController.pnp.culonoreE.php-cs-fixer.cacheNourications//smskecelvea.pngMall//Smskecelveo.phpsms-received.blade.php(©) TeytRelavService.pngphp.pho-cs-rixer.dist.php=.env.productiorphp.phpstorm.meta.php=.phpunit.result.cacheE .prettierignore= windsurtrulesphpde neloer.onophpide heloer_models.ohophp artisanM'CLAUDE.mdcomooser.isoncomooser.lockdependency-checker.json8 dev ison= ids tytl=infection.ison.distMLINSTAlI.mdMLINTERNAI WERHOOK SFTUP md=jiminny_storageMiliconsos mdllM Makefile# package-lock.jsonE phpstan.neon.distE phostan-baseline.neon<> phounit.xml60 PTe raw sal query.saM+ README.mdsô sonar-proiect propertiesE test.ov‹> Untitled Diaaram.xmli68usvetur.confio.isM+ WEBHOOK FILTERING_IMPLEMENTATION.mo>1h External Librariesv =° Scratches and Consoles71 %)v batabase Consoles91VSv AEU# console (EUlOSASI DEAL RISKS (SU1A DITEUl110 dSA EU (EU]v Ajiminny@localhostA console (jiminny@localhost]# Di [liminny@localhostlA HS local [iiminny@localhostlA SF ['iiminny@localhost]& zoho dev [iiminny@localhost]V & PRODA console [PROD]& console 1 [PROD1125 >138 >LDIPRODclass SmsRecelvedlest extends Testcaseprotected function setUp(): voiduunwvunsav,Sthis->activity=>prospect_name = self::PROSPECT_NAME;Sthis-›activity=>from = Sthis->fromParticipant;Sthis->activity->to = $this->toParticipant;$this->outboundActivity = Mockery::mock( ...args: Activity::class)->shouldIgnoreMissing0:Config::set('jiminny.google_text_host', self::G00GLE_TEXT_HOST);contig..seulaminny.deploy-regzon, "us)protected function tearDown: voidparent:: tearbownonubuic tunction testlonstructorstores.ctvirvo: vozdSnotification = new SmsReceivedSthis->activitv)*Sreflection = new Reflectionclass(Snotification);Sproperty = Sreflection->getProverty nan'activity'):soronertv->setAccessihl.er accSthis->assertSame(Sthis->activitv. Soronertv->aetValue(Snotification)):public function testViaReturnsMail: voidf...}public function testToMailReturnsMailable: void{...}public function testToMailWithUsRegion: void{...}public function testToMailWithEuRegion: void{...}public function testGetOutboundActivityReturnsNullWhenNo0utboundFound0: void{...}public function testToMailCallsGetOutboundActivitv@: voidf...}=custom.log= laravel.logA SF [jiminny@localhost]4 HS_local [jiminny@localhost]& console [PROD] X# console [euyA console [STAGING]66203A3 A12Av 663664665667678— 680- - 699700701Tx: AutovSo jiminny v•ts where id = 54;SELECT * FROM users WHERE id IN (24623,29443,29613);041 A1 A40 V65 ^SELECT * FROM automated_report_results WHERE uuid to_ bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid:select * from text relays where created at › '2026-01-01'*and id IN (32415, 32416);# and id = 32412:select * From users where team 1d = 2 and emall Like '%scott% and 10 = 2951101TITTT 11SELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436SELECT email provider id. COUNT*) as count. GROUP CONCATGid) as ids. GROUP CONCAT(status) as statusesFROM text_relaysWHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')GROUP BY email_provider_id;SELECT id, status, telephony_provider_id, created_atFROM activitiesWHERE id IN (80028719, 80028846)SELECT id, status, code, email_sent_at, created_at, updated_atSDnM tov+ nolaveWHERE id IN (32415, 32416);SELECT id, status,code, sender, recipient, created_atFROM text relaysWHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'ORDER BY created at DESCLIMIT 10:SELECT id, uuid, status, code, sender, recipient, created at. updated_ atFROM text relavsWHERE uuid = uuid to bin('0626141c-27a6-4d8c-aff8-7c8020a2c656') :TTT TTTTTSELECT DISTINCT U.id. U.email, U.name, u.softohone number, couNta.id as sms count.FROM users uINNER JOIN activities a 1<->1..n: ON u.id = a.user idWHERE a.type LIKE "sms%AND a.crpated at > NATE SUR(NOWO. TNTERVAI 30 DAY)GROUP BY u.id, u.email, u.name, u.softphone_numberORDER RY sms count nESC.select * from teams whene id = 11colont + fnom noloc,Cascade•Code Review: Email# Al chapter • 49m left100% S2• Wed 20 May 17:11:24SmsReceivedTest v+0 ..ImplementAsk anvthing (*4LW Windsurf Teams 678:53 UTF-8Aensod...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64671
|
2264
|
24
|
2026-05-20T14:11:19.038979+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286279038_m1.jpg...
|
PhpStorm
|
faVsco.js – console [PROD]
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65...
|
[{"role":"AXButton","text" [{"role":"AXButton","text":"Project: faVsco.js, menu","depth":5,"on_screen":true,"help_text":"~/jiminny/app","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"#12105 on JY-20915-add-domain-specific-emai…relay, menu","depth":5,"on_screen":true,"help_text":"Pull request #12105 exists for current branch JY-20915-add-domain-specific-emai…relay","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Start Listening for PHP Debug Connections","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"SmsReceivedTest","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Run 'SmsReceivedTest'","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Debug 'SmsReceivedTest'","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"More Actions","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"JetBrains AI","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Search Everywhere","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"IDE and Project Settings","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.088194445,"height":0.027777778},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"3","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"3","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","depth":4,"on_screen":true,"value":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Execute","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Explain Plan","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Browse Query History","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"View Parameters","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Open Query Execution Settings…","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"In-Editor Results","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Tx: Auto","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Cancel Running Statements","depth":4,"on_screen":true,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Playground","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"jiminny","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.088194445,"height":0.027777778},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"41","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"1","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"40","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"65","depth":4,"on_screen":true,"role_description":"text"}]...
|
8012156536030131395
|
-6961192827286815854
|
click
|
accessibility
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64670
|
2265
|
51
|
2026-05-20T14:11:09.686051+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286269686_m2.jpg...
|
PhpStorm
|
faVsco.js – console [PROD]
|
1
|
NULL
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993
SELECT * FROM users WHERE id = 25061;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 994;
SELECT * FROM crm_profiles WHERE user_id = 25061;
select * from crm_configurations where id = 834;
SELECT * FROM teams WHERE id = 882;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;
SELECT * FROM contacts where crm_configuration_id = 834;
SELECT * FROM opportunities WHERE team_id = 933
# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');
AND id IN (8482561,18352941,19042734,19232139,19445140,19472541);
SELECT * FROM opportunity_contacts
WHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 485; #
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
select crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id
where crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')
# and l.converted_at IS NOT NULL
;
# [PASSWORD_DOTS]
SELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')
and opportunity_id IS NULL
order by id desc;
SELECT * FROM teams WHERE id = 604; # 598
SELECT * FROM activities WHERE id = 74410828; # [EMAIL]
SELECT * FROM accounts WHERE id = 20068382;
SELECT * FROM accounts WHERE id = 35186038;
SELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 559 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;
select * from sidekick_settings where team_id = 781;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100
SELECT * FROM crm_layouts WHERE crm_configuration_id = 711;
SELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL
and is_internal = 0 and status = 'completed'
order by id desc;
SELECT * FROM crm_layout_entities
WHERE crm_layout_id IN (2352, 2353);
;
SELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 556 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;
SELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;
select * from contacts
where crm_configuration_id = 530
and crm_provider_id = 872252;
select * from activities where crm_configuration_id = 530
and user_id = 14343 and type like '%softphone%'
and created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);
SELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t
JOIN crm_configurations c ON t.id = c.team_id
WHERE t.status = 'active';
SELECT * FROM teams where id = 1091;
SELECT * FROM crm_configurations where team_id = 1091;
SELECT * FROM activity_providers where team_id = 1091;
SELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT * FROM teams WHERE name LIKE '%Leadventure%';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1091 and sa.provider = 'salesforce';
SELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812
SELECT * FROM teams where id = 862;
SELECT * FROM crm_configurations where team_id = 862;
SELECT * FROM activity_providers where team_id = 862;
SELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT t.id, crm.id, crm.provider, ap.* FROM teams t
join crm_configurations crm on t.id = crm.team_id
join activity_providers ap on t.id = ap.team_id
where t.status = 'active' and ap.is_enabled = 1
and crm.provider = 'hubspot'
and ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',
'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');
SELECT * FROM teams where id = 1068;
SELECT * FROM crm_configurations where team_id = 1068;
SELECT * FROM activity_providers where team_id = 1068;
SELECT * FROM activities a
where crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')
and a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'
)
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by a.id desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1068 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262
SELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
select * from crm_layouts where crm_configuration_id = 834;
select * from crm_layout_entities where crm_layout_id = 2780;
select * from crm_fields where id IN (321153,321192,321193,321194);
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1057 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8
SELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20
SELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10
SELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #
SELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;
select * from users where team_id = 51; # 7783
SELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130
select * from activity_searches where user_id = 7783;
select * from activity_search_filters where activity_search_id IN (32291, 32292);
SELECT asf.activity_search_id, asf.id, asf.value
FROM activity_search_filters asf
WHERE asf.filter = 'group_id'
AND asf.value IN (
SELECT CONCAT(
HEX(SUBSTR(uuid, 5, 4)), '-',
HEX(SUBSTR(uuid, 3, 2)), '-',
HEX(SUBSTR(uuid, 1, 2)), '-',
HEX(SUBSTR(uuid, 9, 2)), '-',
HEX(SUBSTR(uuid, 11))
)
FROM groups
WHERE deleted_at IS NOT NULL
);
SELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where provider = 'hubspot';
SELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133
SELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null
# [PASSWORD_DOTS]
select * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';
select
cp.*
# DISTINCT t.id
# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields
FROM crm_profiles cp
JOIN crm_configurations crm on crm.id = cp.crm_configuration_id
JOIN users u on u.id = cp.user_id
JOIN teams t ON t.id = crm.team_id
WHERE crm.provider = 'salesforce' and t.status = 'active'
and cp.archived_at IS NULL and u.deleted_at IS NULL
and t.id NOT IN (1093)
and t.id = 2
and cp.contact_fields IS NULL;
# and c.crm_provider_id = '003Uu00000ojD4NIAU';
SELECT * FROM users WHERE id = 26484;
SELECT * FROM crm_profiles WHERE user_id = 26484;
SELECT * FROM social_accounts WHERE sociable_id = 26484;
SELECT * FROM crm_configurations where provider = 'salesforce';
select * from users where id IN (10022, 10403);
select * from users where team_id IN (526);
select * from teams where id IN (526, 532);
select * from crm_configurations where id IN (500, 516);
select * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);
select * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 526 and sa.provider = 'salesforce';
select * from team_settings where team_id IN (526, 532);
select * from users where id IN (22824);
select * from crm_profiles where crm_configuration_id IN (1026);
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1093 and sa.provider = 'salesforce';
select * from teams where id = 1099;
select * from users where id = 29643
select * from activity_processing_states;
SELECT * FROM teams where name LIKE '%Fare%'; # 233
SELECT * FROM opportunities where crm_configuration_id = 215
# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'
;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1088 and sa.provider = 'hubspot';
SELECT * FROM teams order by updated_at DESC
SELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account
select * from crm_configurations where provider = 'pipedrive';
select * from teams where id = 957;
select * from crm_configurations where id = 957;
SELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743
SELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;
select * from users where team_id = 1; # 26726 - Gabriela Dureva
SELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific
select * from activities where user_id = 26726 order by id desc;
select * from contacts where crm_configuration_id = 1
and email IN ('[EMAIL]', '[EMAIL]'); # 2094416, 2093620
SELECT * FROM contacts WHERE id = 6284931;
SELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id
WHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;
select * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);
select * from crm_configurations where id = 1;
43801692-1aeb-32ce-acba-5b80a479701a
44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b
405975c0-b3d0-7aaa-821f-09d59cae6dd1
4caf848d-4bed-2299-b248-7788d41f9fca
49bedc3f-f196-eef3-89c3-dea6a3b4aa63
43420989-a09d-b8f8-9806-c8bbf7a02aac
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
SELECT * FROM activities WHERE id = 75461988;
SELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;
select * from contacts where id = 17900517;
select * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id
where crm.provider != 'salesforce';
select * from users where id = 21047;
SELECT * FROM crm_configurations WHERE id = 892;
SELECT * FROM teams WHERE id = 942;
select * from opportunities where team_id = 942 order by updated_at desc;
select * from contacts where team_id = 942 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 942 and sa.provider = 'hubspot';
SELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430
SELECT * FROM crm_configurations WHERE id = 1;
SELECT * FROM teams WHERE crm_id = 1;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
select id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1
SELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430
select * from teams where id = 852;
select * from groups where id = 2286;
select * from sidekick_settings where team_id = 852;
select * from default_activity_types where team_id = 852;
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1 AND u.deleted_at IS NULL
AND u.crm_required = 1
AND u.team_id = 1
ORDER BY u.team_id;
SELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (
18481
);
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1
AND u.deleted_at IS NULL
AND u.crm_required = 1
# AND u.team_id = 1
AND p.id IS NULL -- Move this condition to WHERE clause
ORDER BY u.team_id;
SELECT * FROM opportunities WHERE id = 20002609;
select * from teams where id = 1122; # Velatir, 29953 - [EMAIL]
select * from crm_configurations where id = 1060;
select * from crm_layouts where crm_configuration_id = 1060;
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1122 and sa.provider = 'hubspot';
select * from opportunities where team_id = 1122 order by updated_at desc;
select * from crm_field_data where object_type = 'contact';
SELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 248 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS
SELECT * FROM users where id = 24115;
SELECT * FROM accounts where id = 4002896;
SELECT * FROM teams WHERE name LIKE '%adswerve%';
SELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN ("0069N000003GIQ9QAO","0061r000019yGP9AAM","0066900001S2KWlAAN","0066900001TDpj2AAD","0066900001b8uEwAAI","0069N000001rQi0QAE","006QF00000KD40mYAD","006QF00000LzpRJYAZ","0069N000002uomtQAA","0069N000002xlMLQAY","0066900001NV6ubAAD","0061r00001HJp45AAD","006QF00000uTlUoYAK","006QF00000v0bZqYAI");
SELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203
SELECT u.id, u.email, ac.name, a.* FROM activities a
JOIN users u ON a.user_id = u.id
JOIN accounts ac ON a.account_id = ac.id
WHERE
uuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or
uuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or
uuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;
select * from users where id = 5825;
SELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;
select * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;
19594, 862
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 862 and sa.provider = 'salesforce';
select * from automated_reports where id = 36;
select ar.frequency, r.*, ar.* from automated_report_results r
join automated_reports ar on r.report_id = ar.id
where ar.frequency != 'one_off';
select s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;
select * from nudges n where n.activity_search_id
select * from teams where created_at > '2026-03-09';
SELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;
select * from users where team_id = 1 and name like '%Lukas%'; # 7160
SELECT * FROM teams WHERE id = 575;
select * from opportunities where team_id = 575;
SELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,
select * from opportunities where team_id = 1126;
SELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,
select * from opportunities where team_id = 1125;
select * from contacts c
where c.team_id = 882;
SELECT * FROM activities WHERE id = 76822967;
SELECT * FROM crm_profiles WHERE user_id = 15440;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 555;
SELECT * FROM crm_configurations WHERE id = 555;
SELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 581 and sa.provider = 'salesforce';
SELECT * FROM automated_report_results order by id desc;
select * from features;
select * from team_features where feature_id = 40;
select * from teams where id = 556;
select * from automated_reports;
where id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , ["pdf","podcast"]
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from automated_report_results order by id desc;
SELECT * FROM automated_report_results WHERE id = 1919;
select * from automated_report_results WHERE report_id = 54;
select * from opportunities where id = 7594349;
SELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - [EMAIL]
select * from playbooks where team_id = 711; # event 226147
SELECT * FROM playbook_categories WHERE playbook_id = 5515;
SELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';
SELECT * FROM crm_fields WHERE id = 226147;
SELECT * FROM crm_field_values WHERE crm_field_id = 226147;
SELECT * FROM crm_configurations WHERE id = 692;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 711 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;
select * from leads;
select * from calendars;
SELECT
t.id AS team_id,
t.name,
LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain
FROM teams t
JOIN users u ON u.team_id = t.id
JOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'
LEFT JOIN team_domains td
ON td.team_id = t.id
AND td.deleted_at IS NULL
AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))
GROUP BY t.id, t.name, calendar_domain
ORDER BY t.name, calendar_domain;
select * from users u join calendars c on c.user_id = u.id
where u.team_id = 882;
select * from activities where id = 74049485; # team 563 crm 537
select * from activities where id = 73272382; # team 563 crm 537
select * from activities where id = 64400389; # team 563 crm 537
select * from activities where id = 58081273; # team 563 crm 537
select * from activities where id = 54520297; # team 563 crm 537
select * from participants where activity_id = 58081273;
select * from activities where crm_configuration_id = 537 and provider = 'aircall'
and account_id = 19003658 order by updated_at desc;
select * from contacts where crm_configuration_id = 537 and id = 35957759;
select * from accounts where crm_configuration_id = 537 and id = 19003658;
select * from automated_report_results where id = 1976;
select * from automated_reports where id = 583;
select * from activity_searches where id = 87714;
select * from activity_search_filters where activity_search_id = 87714;
SELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid
or uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;
SELECT * FROM crm_configurations WHERE provider = 'hubspot';
select * from rate_limits;
select * from automated_report_results where media_type = 'pdf' and status = 2
and id IN (18, 1872);
select * from automated_reports where id = 54;
SELECT * FROM users WHERE id IN (24623,29443,29613);
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from text_relays where created_at > '2026-01-01';
and id IN (32415, 32416);
# and id = 32412;
select * from users where team_id = 2 and email like '%scott%' and id = 29510;
SELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436
SELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses
FROM text_relays
WHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')
GROUP BY email_provider_id;
SELECT id, status, telephony_provider_id, created_at
FROM activities
WHERE id IN (80028719, 80028846);
SELECT id, status, code, email_sent_at, created_at, updated_at
FROM text_relays
WHERE id IN (32415, 32416);
SELECT id, status, code, sender, recipient, created_at
FROM text_relays
WHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'
ORDER BY created_at DESC
LIMIT 10;
SELECT id, uuid, status, code, sender, recipient, created_at, updated_at
FROM text_relays
WHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');
# [PASSWORD_DOTS]
SELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count
FROM users u
INNER JOIN activities a ON u.id = a.user_id
WHERE a.type LIKE 'sms%'
AND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)
GROUP BY u.id, u.email, u.name, u.softphone_number
ORDER BY sms_count DESC;
select * from teams where id = 1;
select * from roles;
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
[{"role":"AXButton","text" [{"role":"AXButton","text":"Project: faVsco.js, menu","depth":5,"bounds":{"left":0.025930852,"top":0.019952115,"width":0.03856383,"height":0.025538707},"on_screen":true,"help_text":"~/jiminny/app","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"#12105 on JY-20915-add-domain-specific-emai…relay, menu","depth":5,"bounds":{"left":0.064494684,"top":0.019952115,"width":0.124667555,"height":0.025538707},"on_screen":true,"help_text":"Pull request #12105 exists for current branch JY-20915-add-domain-specific-emai…relay","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Start Listening for PHP Debug Connections","depth":5,"bounds":{"left":0.8450798,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"SmsReceivedTest","depth":6,"bounds":{"left":0.86037236,"top":0.019952115,"width":0.05518617,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Run 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9155585,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Debug 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9268617,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"More Actions","depth":6,"bounds":{"left":0.9381649,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"JetBrains AI","depth":5,"bounds":{"left":0.96609044,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Search Everywhere","depth":5,"bounds":{"left":0.9773936,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"IDE and Project Settings","depth":5,"bounds":{"left":0.9886968,"top":0.019952115,"width":0.011303186,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.042220745,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"3","depth":4,"bounds":{"left":0.38098404,"top":0.15003991,"width":0.007978723,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"3","depth":4,"bounds":{"left":0.39095744,"top":0.15003991,"width":0.007978723,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12","depth":4,"bounds":{"left":0.40093085,"top":0.15003991,"width":0.009640957,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"bounds":{"left":0.41223404,"top":0.14844373,"width":0.00731383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"bounds":{"left":0.4195479,"top":0.14844373,"width":0.006981383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","depth":4,"on_screen":true,"value":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Execute","depth":4,"bounds":{"left":0.42785904,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Explain Plan","depth":4,"bounds":{"left":0.43650267,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Browse Query History","depth":4,"bounds":{"left":0.4474734,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"View Parameters","depth":4,"bounds":{"left":0.45611703,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Open Query Execution Settings…","depth":4,"bounds":{"left":0.46476063,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"In-Editor Results","depth":4,"bounds":{"left":0.47573137,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Tx: Auto","depth":4,"bounds":{"left":0.4867021,"top":0.09896249,"width":0.024268618,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Cancel Running Statements","depth":4,"bounds":{"left":0.51329786,"top":0.09896249,"width":0.008643617,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Playground","depth":4,"bounds":{"left":0.5242686,"top":0.09896249,"width":0.029587766,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"jiminny","depth":4,"bounds":{"left":0.70611703,"top":0.09896249,"width":0.02825798,"height":0.01915403},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.042220745,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"41","depth":4,"bounds":{"left":0.6761968,"top":0.123703115,"width":0.009640957,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"1","depth":4,"bounds":{"left":0.6878325,"top":0.123703115,"width":0.00731383,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"40","depth":4,"bounds":{"left":0.69714093,"top":0.123703115,"width":0.010305851,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"65","depth":4,"bounds":{"left":0.7094415,"top":0.123703115,"width":0.010305851,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"bounds":{"left":0.72140956,"top":0.12210695,"width":0.00731383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"bounds":{"left":0.7287234,"top":0.12210695,"width":0.006981383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993\nSELECT * FROM users WHERE id = 25061;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 994;\nSELECT * FROM crm_profiles WHERE user_id = 25061;\n\nselect * from crm_configurations where id = 834;\nSELECT * FROM teams WHERE id = 882;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;\n\nSELECT * FROM contacts where crm_configuration_id = 834;\nSELECT * FROM opportunities WHERE team_id = 933\n# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');\nAND id IN (8482561,18352941,19042734,19232139,19445140,19472541);\nSELECT * FROM opportunity_contacts\nWHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 485; #\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nselect crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id\nwhere crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')\n# and l.converted_at IS NOT NULL\n;\n\n# ********************************************************************\nSELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')\nand opportunity_id IS NULL\norder by id desc;\n\nSELECT * FROM teams WHERE id = 604; # 598\nSELECT * FROM activities WHERE id = 74410828; # chelseaw@allvoices.co\nSELECT * FROM accounts WHERE id = 20068382;\nSELECT * FROM accounts WHERE id = 35186038;\n\nSELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 559 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;\nselect * from sidekick_settings where team_id = 781;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100\n\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 711;\nSELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL\nand is_internal = 0 and status = 'completed'\norder by id desc;\n\nSELECT * FROM crm_layout_entities\nWHERE crm_layout_id IN (2352, 2353);\n;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 556 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;\nSELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;\nselect * from contacts\nwhere crm_configuration_id = 530\nand crm_provider_id = 872252;\n\nselect * from activities where crm_configuration_id = 530\nand user_id = 14343 and type like '%softphone%'\nand created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);\n\n\nSELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t\nJOIN crm_configurations c ON t.id = c.team_id\nWHERE t.status = 'active';\n\nSELECT * FROM teams where id = 1091;\nSELECT * FROM crm_configurations where team_id = 1091;\nSELECT * FROM activity_providers where team_id = 1091;\nSELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT * FROM teams WHERE name LIKE '%Leadventure%';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1091 and sa.provider = 'salesforce';\n\nSELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812\nSELECT * FROM teams where id = 862;\nSELECT * FROM crm_configurations where team_id = 862;\nSELECT * FROM activity_providers where team_id = 862;\nSELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT t.id, crm.id, crm.provider, ap.* FROM teams t\njoin crm_configurations crm on t.id = crm.team_id\njoin activity_providers ap on t.id = ap.team_id\nwhere t.status = 'active' and ap.is_enabled = 1\nand crm.provider = 'hubspot'\nand ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',\n 'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');\n\nSELECT * FROM teams where id = 1068;\nSELECT * FROM crm_configurations where team_id = 1068;\nSELECT * FROM activity_providers where team_id = 1068;\n\nSELECT * FROM activities a\nwhere crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')\nand a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'\n )\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by a.id desc;\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1068 and sa.provider = 'hubspot';\n\n# ********************************************************************\n# ********************************************************************\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262\nSELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\nselect * from crm_layouts where crm_configuration_id = 834;\nselect * from crm_layout_entities where crm_layout_id = 2780;\nselect * from crm_fields where id IN (321153,321192,321193,321194);\n\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1057 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8\n\nSELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20\n\nSELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10\n\nSELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #\n\nSELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;\nselect * from users where team_id = 51; # 7783\nSELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130\nselect * from activity_searches where user_id = 7783;\nselect * from activity_search_filters where activity_search_id IN (32291, 32292);\n\nSELECT asf.activity_search_id, asf.id, asf.value\nFROM activity_search_filters asf\nWHERE asf.filter = 'group_id'\nAND asf.value IN (\n SELECT CONCAT(\n HEX(SUBSTR(uuid, 5, 4)), '-',\n HEX(SUBSTR(uuid, 3, 2)), '-',\n HEX(SUBSTR(uuid, 1, 2)), '-',\n HEX(SUBSTR(uuid, 9, 2)), '-',\n HEX(SUBSTR(uuid, 11))\n )\n FROM groups\n WHERE deleted_at IS NOT NULL\n);\n\nSELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th\n# ********************************************************************\nSELECT * FROM crm_configurations where provider = 'hubspot';\nSELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133\nSELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null\n# ********************************************************************\n\nselect * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';\nselect\n cp.*\n# DISTINCT t.id\n# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields\nFROM crm_profiles cp\nJOIN crm_configurations crm on crm.id = cp.crm_configuration_id\nJOIN users u on u.id = cp.user_id\nJOIN teams t ON t.id = crm.team_id\nWHERE crm.provider = 'salesforce' and t.status = 'active'\n and cp.archived_at IS NULL and u.deleted_at IS NULL\n and t.id NOT IN (1093)\n and t.id = 2\n and cp.contact_fields IS NULL;\n# and c.crm_provider_id = '003Uu00000ojD4NIAU';\n\nSELECT * FROM users WHERE id = 26484;\nSELECT * FROM crm_profiles WHERE user_id = 26484;\nSELECT * FROM social_accounts WHERE sociable_id = 26484;\nSELECT * FROM crm_configurations where provider = 'salesforce';\nselect * from users where id IN (10022, 10403);\nselect * from users where team_id IN (526);\nselect * from teams where id IN (526, 532);\nselect * from crm_configurations where id IN (500, 516);\nselect * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);\nselect * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 526 and sa.provider = 'salesforce';\nselect * from team_settings where team_id IN (526, 532);\n\nselect * from users where id IN (22824);\nselect * from crm_profiles where crm_configuration_id IN (1026);\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1093 and sa.provider = 'salesforce';\n\nselect * from teams where id = 1099;\nselect * from users where id = 29643\n\nselect * from activity_processing_states;\n\nSELECT * FROM teams where name LIKE '%Fare%'; # 233\nSELECT * FROM opportunities where crm_configuration_id = 215\n# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'\n;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1088 and sa.provider = 'hubspot';\n\nSELECT * FROM teams order by updated_at DESC\nSELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account\n\nselect * from crm_configurations where provider = 'pipedrive';\n\nselect * from teams where id = 957;\nselect * from crm_configurations where id = 957;\n\nSELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743\nSELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;\n\nselect * from users where team_id = 1; # 26726 - Gabriela Dureva\nSELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific\nselect * from activities where user_id = 26726 order by id desc;\nselect * from contacts where crm_configuration_id = 1\nand email IN ('charlotte.ward@prolific.com', 'frankie.bryant@prolific.com'); # 2094416, 2093620\nSELECT * FROM contacts WHERE id = 6284931;\n\nSELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id\nWHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;\n\nselect * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);\nselect * from crm_configurations where id = 1;\n\n43801692-1aeb-32ce-acba-5b80a479701a\n44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b\n405975c0-b3d0-7aaa-821f-09d59cae6dd1\n4caf848d-4bed-2299-b248-7788d41f9fca\n49bedc3f-f196-eef3-89c3-dea6a3b4aa63\n43420989-a09d-b8f8-9806-c8bbf7a02aac\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nSELECT * FROM activities WHERE id = 75461988;\n\nSELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;\n\nselect * from contacts where id = 17900517;\n\nselect * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id\nwhere crm.provider != 'salesforce';\n\nselect * from users where id = 21047;\nSELECT * FROM crm_configurations WHERE id = 892;\nSELECT * FROM teams WHERE id = 942;\nselect * from opportunities where team_id = 942 order by updated_at desc;\nselect * from contacts where team_id = 942 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 942 and sa.provider = 'hubspot';\n\nSELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430\nSELECT * FROM crm_configurations WHERE id = 1;\nSELECT * FROM teams WHERE crm_id = 1;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nselect id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1\nSELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430\n\nselect * from teams where id = 852;\nselect * from groups where id = 2286;\nselect * from sidekick_settings where team_id = 852;\nselect * from default_activity_types where team_id = 852;\n\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1 AND u.deleted_at IS NULL\nAND u.crm_required = 1\nAND u.team_id = 1\nORDER BY u.team_id;\n\nSELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (\n18481\n );\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1\n AND u.deleted_at IS NULL\n AND u.crm_required = 1\n# AND u.team_id = 1\n AND p.id IS NULL -- Move this condition to WHERE clause\nORDER BY u.team_id;\n\nSELECT * FROM opportunities WHERE id = 20002609;\nselect * from teams where id = 1122; # Velatir, 29953 - christian@velatir.com\nselect * from crm_configurations where id = 1060;\nselect * from crm_layouts where crm_configuration_id = 1060;\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1122 and sa.provider = 'hubspot';\nselect * from opportunities where team_id = 1122 order by updated_at desc;\n\nselect * from crm_field_data where object_type = 'contact';\n\nSELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 248 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS\nSELECT * FROM users where id = 24115;\nSELECT * FROM accounts where id = 4002896;\nSELECT * FROM teams WHERE name LIKE '%adswerve%';\nSELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN (\"0069N000003GIQ9QAO\",\"0061r000019yGP9AAM\",\"0066900001S2KWlAAN\",\"0066900001TDpj2AAD\",\"0066900001b8uEwAAI\",\"0069N000001rQi0QAE\",\"006QF00000KD40mYAD\",\"006QF00000LzpRJYAZ\",\"0069N000002uomtQAA\",\"0069N000002xlMLQAY\",\"0066900001NV6ubAAD\",\"0061r00001HJp45AAD\",\"006QF00000uTlUoYAK\",\"006QF00000v0bZqYAI\");\nSELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203\n\nSELECT u.id, u.email, ac.name, a.* FROM activities a\nJOIN users u ON a.user_id = u.id\nJOIN accounts ac ON a.account_id = ac.id\nWHERE\nuuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or\nuuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or\nuuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;\n\nselect * from users where id = 5825;\nSELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;\n\nselect * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;\n19594, 862\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 862 and sa.provider = 'salesforce';\n\nselect * from automated_reports where id = 36;\nselect ar.frequency, r.*, ar.* from automated_report_results r\njoin automated_reports ar on r.report_id = ar.id\nwhere ar.frequency != 'one_off';\n\nselect s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;\nselect * from nudges n where n.activity_search_id\n\nselect * from teams where created_at > '2026-03-09';\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;\n\nselect * from users where team_id = 1 and name like '%Lukas%'; # 7160\n\nSELECT * FROM teams WHERE id = 575;\nselect * from opportunities where team_id = 575;\nSELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,\nselect * from opportunities where team_id = 1126;\nSELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,\nselect * from opportunities where team_id = 1125;\nselect * from contacts c\nwhere c.team_id = 882;\n\nSELECT * FROM activities WHERE id = 76822967;\nSELECT * FROM crm_profiles WHERE user_id = 15440;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 555;\nSELECT * FROM crm_configurations WHERE id = 555;\nSELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 581 and sa.provider = 'salesforce';\n\nSELECT * FROM automated_report_results order by id desc;\n\nselect * from features;\nselect * from team_features where feature_id = 40;\n\nselect * from teams where id = 556;\n\nselect * from automated_reports;\nwhere id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , [\"pdf\",\"podcast\"]\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\nselect * from automated_report_results order by id desc;\nSELECT * FROM automated_report_results WHERE id = 1919;\n\nselect * from automated_report_results WHERE report_id = 54;\n\nselect * from opportunities where id = 7594349;\n\nSELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - jiminnyintegration@lesmills.com\nselect * from playbooks where team_id = 711; # event 226147\nSELECT * FROM playbook_categories WHERE playbook_id = 5515;\nSELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';\nSELECT * FROM crm_fields WHERE id = 226147;\nSELECT * FROM crm_field_values WHERE crm_field_id = 226147;\n\nSELECT * FROM crm_configurations WHERE id = 692;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 711 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;\n\nselect * from leads;\n\nselect * from calendars;\n\nSELECT\n t.id AS team_id,\n t.name,\n LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain\nFROM teams t\nJOIN users u ON u.team_id = t.id\nJOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'\nLEFT JOIN team_domains td\n ON td.team_id = t.id\n AND td.deleted_at IS NULL\n AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))\nGROUP BY t.id, t.name, calendar_domain\nORDER BY t.name, calendar_domain;\n\nselect * from users u join calendars c on c.user_id = u.id\nwhere u.team_id = 882;\n\n\nselect * from activities where id = 74049485; # team 563 crm 537\nselect * from activities where id = 73272382; # team 563 crm 537\nselect * from activities where id = 64400389; # team 563 crm 537\nselect * from activities where id = 58081273; # team 563 crm 537\nselect * from activities where id = 54520297; # team 563 crm 537\nselect * from participants where activity_id = 58081273;\n\nselect * from activities where crm_configuration_id = 537 and provider = 'aircall'\nand account_id = 19003658 order by updated_at desc;\n\nselect * from contacts where crm_configuration_id = 537 and id = 35957759;\nselect * from accounts where crm_configuration_id = 537 and id = 19003658;\n\nselect * from automated_report_results where id = 1976;\nselect * from automated_reports where id = 583;\nselect * from activity_searches where id = 87714;\nselect * from activity_search_filters where activity_search_id = 87714;\n\nSELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid\nor uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot';\nselect * from rate_limits;\n\nselect * from automated_report_results where media_type = 'pdf' and status = 2\nand id IN (18, 1872);\nselect * from automated_reports where id = 54;\nSELECT * FROM users WHERE id IN (24623,29443,29613);\n\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\n\nselect * from text_relays where created_at > '2026-01-01';\nand id IN (32415, 32416);\n# and id = 32412;\n\nselect * from users where team_id = 2 and email like '%scott%' and id = 29510;\n\nSELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436\n\nSELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses\nFROM text_relays\nWHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')\nGROUP BY email_provider_id;\nSELECT id, status, telephony_provider_id, created_at\nFROM activities\nWHERE id IN (80028719, 80028846);\nSELECT id, status, code, email_sent_at, created_at, updated_at\nFROM text_relays\nWHERE id IN (32415, 32416);\nSELECT id, status, code, sender, recipient, created_at\nFROM text_relays\nWHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'\nORDER BY created_at DESC\nLIMIT 10;\n\nSELECT id, uuid, status, code, sender, recipient, created_at, updated_at\nFROM text_relays\nWHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');\n\n# ***************\nSELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count\nFROM users u\nINNER JOIN activities a ON u.id = a.user_id\nWHERE a.type LIKE 'sms%'\nAND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)\nGROUP BY u.id, u.email, u.name, u.softphone_number\nORDER BY sms_count DESC;\n\nselect * from teams where id = 1;\n\nselect * from roles;","depth":4,"on_screen":true,"value":"SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993\nSELECT * FROM users WHERE id = 25061;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 994;\nSELECT * FROM crm_profiles WHERE user_id = 25061;\n\nselect * from crm_configurations where id = 834;\nSELECT * FROM teams WHERE id = 882;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;\n\nSELECT * FROM contacts where crm_configuration_id = 834;\nSELECT * FROM opportunities WHERE team_id = 933\n# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');\nAND id IN (8482561,18352941,19042734,19232139,19445140,19472541);\nSELECT * FROM opportunity_contacts\nWHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 485; #\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nselect crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id\nwhere crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')\n# and l.converted_at IS NOT NULL\n;\n\n# ********************************************************************\nSELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')\nand opportunity_id IS NULL\norder by id desc;\n\nSELECT * FROM teams WHERE id = 604; # 598\nSELECT * FROM activities WHERE id = 74410828; # chelseaw@allvoices.co\nSELECT * FROM accounts WHERE id = 20068382;\nSELECT * FROM accounts WHERE id = 35186038;\n\nSELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 559 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;\nselect * from sidekick_settings where team_id = 781;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100\n\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 711;\nSELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL\nand is_internal = 0 and status = 'completed'\norder by id desc;\n\nSELECT * FROM crm_layout_entities\nWHERE crm_layout_id IN (2352, 2353);\n;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 556 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;\nSELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;\nselect * from contacts\nwhere crm_configuration_id = 530\nand crm_provider_id = 872252;\n\nselect * from activities where crm_configuration_id = 530\nand user_id = 14343 and type like '%softphone%'\nand created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);\n\n\nSELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t\nJOIN crm_configurations c ON t.id = c.team_id\nWHERE t.status = 'active';\n\nSELECT * FROM teams where id = 1091;\nSELECT * FROM crm_configurations where team_id = 1091;\nSELECT * FROM activity_providers where team_id = 1091;\nSELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT * FROM teams WHERE name LIKE '%Leadventure%';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1091 and sa.provider = 'salesforce';\n\nSELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812\nSELECT * FROM teams where id = 862;\nSELECT * FROM crm_configurations where team_id = 862;\nSELECT * FROM activity_providers where team_id = 862;\nSELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT t.id, crm.id, crm.provider, ap.* FROM teams t\njoin crm_configurations crm on t.id = crm.team_id\njoin activity_providers ap on t.id = ap.team_id\nwhere t.status = 'active' and ap.is_enabled = 1\nand crm.provider = 'hubspot'\nand ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',\n 'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');\n\nSELECT * FROM teams where id = 1068;\nSELECT * FROM crm_configurations where team_id = 1068;\nSELECT * FROM activity_providers where team_id = 1068;\n\nSELECT * FROM activities a\nwhere crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')\nand a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'\n )\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by a.id desc;\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1068 and sa.provider = 'hubspot';\n\n# ********************************************************************\n# ********************************************************************\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262\nSELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\nselect * from crm_layouts where crm_configuration_id = 834;\nselect * from crm_layout_entities where crm_layout_id = 2780;\nselect * from crm_fields where id IN (321153,321192,321193,321194);\n\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1057 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8\n\nSELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20\n\nSELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10\n\nSELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #\n\nSELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;\nselect * from users where team_id = 51; # 7783\nSELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130\nselect * from activity_searches where user_id = 7783;\nselect * from activity_search_filters where activity_search_id IN (32291, 32292);\n\nSELECT asf.activity_search_id, asf.id, asf.value\nFROM activity_search_filters asf\nWHERE asf.filter = 'group_id'\nAND asf.value IN (\n SELECT CONCAT(\n HEX(SUBSTR(uuid, 5, 4)), '-',\n HEX(SUBSTR(uuid, 3, 2)), '-',\n HEX(SUBSTR(uuid, 1, 2)), '-',\n HEX(SUBSTR(uuid, 9, 2)), '-',\n HEX(SUBSTR(uuid, 11))\n )\n FROM groups\n WHERE deleted_at IS NOT NULL\n);\n\nSELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th\n# ********************************************************************\nSELECT * FROM crm_configurations where provider = 'hubspot';\nSELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133\nSELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null\n# ********************************************************************\n\nselect * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';\nselect\n cp.*\n# DISTINCT t.id\n# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields\nFROM crm_profiles cp\nJOIN crm_configurations crm on crm.id = cp.crm_configuration_id\nJOIN users u on u.id = cp.user_id\nJOIN teams t ON t.id = crm.team_id\nWHERE crm.provider = 'salesforce' and t.status = 'active'\n and cp.archived_at IS NULL and u.deleted_at IS NULL\n and t.id NOT IN (1093)\n and t.id = 2\n and cp.contact_fields IS NULL;\n# and c.crm_provider_id = '003Uu00000ojD4NIAU';\n\nSELECT * FROM users WHERE id = 26484;\nSELECT * FROM crm_profiles WHERE user_id = 26484;\nSELECT * FROM social_accounts WHERE sociable_id = 26484;\nSELECT * FROM crm_configurations where provider = 'salesforce';\nselect * from users where id IN (10022, 10403);\nselect * from users where team_id IN (526);\nselect * from teams where id IN (526, 532);\nselect * from crm_configurations where id IN (500, 516);\nselect * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);\nselect * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 526 and sa.provider = 'salesforce';\nselect * from team_settings where team_id IN (526, 532);\n\nselect * from users where id IN (22824);\nselect * from crm_profiles where crm_configuration_id IN (1026);\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1093 and sa.provider = 'salesforce';\n\nselect * from teams where id = 1099;\nselect * from users where id = 29643\n\nselect * from activity_processing_states;\n\nSELECT * FROM teams where name LIKE '%Fare%'; # 233\nSELECT * FROM opportunities where crm_configuration_id = 215\n# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'\n;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1088 and sa.provider = 'hubspot';\n\nSELECT * FROM teams order by updated_at DESC\nSELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account\n\nselect * from crm_configurations where provider = 'pipedrive';\n\nselect * from teams where id = 957;\nselect * from crm_configurations where id = 957;\n\nSELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743\nSELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;\n\nselect * from users where team_id = 1; # 26726 - Gabriela Dureva\nSELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific\nselect * from activities where user_id = 26726 order by id desc;\nselect * from contacts where crm_configuration_id = 1\nand email IN ('charlotte.ward@prolific.com', 'frankie.bryant@prolific.com'); # 2094416, 2093620\nSELECT * FROM contacts WHERE id = 6284931;\n\nSELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id\nWHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;\n\nselect * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);\nselect * from crm_configurations where id = 1;\n\n43801692-1aeb-32ce-acba-5b80a479701a\n44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b\n405975c0-b3d0-7aaa-821f-09d59cae6dd1\n4caf848d-4bed-2299-b248-7788d41f9fca\n49bedc3f-f196-eef3-89c3-dea6a3b4aa63\n43420989-a09d-b8f8-9806-c8bbf7a02aac\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nSELECT * FROM activities WHERE id = 75461988;\n\nSELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;\n\nselect * from contacts where id = 17900517;\n\nselect * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id\nwhere crm.provider != 'salesforce';\n\nselect * from users where id = 21047;\nSELECT * FROM crm_configurations WHERE id = 892;\nSELECT * FROM teams WHERE id = 942;\nselect * from opportunities where team_id = 942 order by updated_at desc;\nselect * from contacts where team_id = 942 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 942 and sa.provider = 'hubspot';\n\nSELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430\nSELECT * FROM crm_configurations WHERE id = 1;\nSELECT * FROM teams WHERE crm_id = 1;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nselect id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1\nSELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430\n\nselect * from teams where id = 852;\nselect * from groups where id = 2286;\nselect * from sidekick_settings where team_id = 852;\nselect * from default_activity_types where team_id = 852;\n\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1 AND u.deleted_at IS NULL\nAND u.crm_required = 1\nAND u.team_id = 1\nORDER BY u.team_id;\n\nSELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (\n18481\n );\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1\n AND u.deleted_at IS NULL\n AND u.crm_required = 1\n# AND u.team_id = 1\n AND p.id IS NULL -- Move this condition to WHERE clause\nORDER BY u.team_id;\n\nSELECT * FROM opportunities WHERE id = 20002609;\nselect * from teams where id = 1122; # Velatir, 29953 - christian@velatir.com\nselect * from crm_configurations where id = 1060;\nselect * from crm_layouts where crm_configuration_id = 1060;\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1122 and sa.provider = 'hubspot';\nselect * from opportunities where team_id = 1122 order by updated_at desc;\n\nselect * from crm_field_data where object_type = 'contact';\n\nSELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 248 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS\nSELECT * FROM users where id = 24115;\nSELECT * FROM accounts where id = 4002896;\nSELECT * FROM teams WHERE name LIKE '%adswerve%';\nSELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN (\"0069N000003GIQ9QAO\",\"0061r000019yGP9AAM\",\"0066900001S2KWlAAN\",\"0066900001TDpj2AAD\",\"0066900001b8uEwAAI\",\"0069N000001rQi0QAE\",\"006QF00000KD40mYAD\",\"006QF00000LzpRJYAZ\",\"0069N000002uomtQAA\",\"0069N000002xlMLQAY\",\"0066900001NV6ubAAD\",\"0061r00001HJp45AAD\",\"006QF00000uTlUoYAK\",\"006QF00000v0bZqYAI\");\nSELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203\n\nSELECT u.id, u.email, ac.name, a.* FROM activities a\nJOIN users u ON a.user_id = u.id\nJOIN accounts ac ON a.account_id = ac.id\nWHERE\nuuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or\nuuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or\nuuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;\n\nselect * from users where id = 5825;\nSELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;\n\nselect * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;\n19594, 862\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 862 and sa.provider = 'salesforce';\n\nselect * from automated_reports where id = 36;\nselect ar.frequency, r.*, ar.* from automated_report_results r\njoin automated_reports ar on r.report_id = ar.id\nwhere ar.frequency != 'one_off';\n\nselect s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;\nselect * from nudges n where n.activity_search_id\n\nselect * from teams where created_at > '2026-03-09';\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;\n\nselect * from users where team_id = 1 and name like '%Lukas%'; # 7160\n\nSELECT * FROM teams WHERE id = 575;\nselect * from opportunities where team_id = 575;\nSELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,\nselect * from opportunities where team_id = 1126;\nSELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,\nselect * from opportunities where team_id = 1125;\nselect * from contacts c\nwhere c.team_id = 882;\n\nSELECT * FROM activities WHERE id = 76822967;\nSELECT * FROM crm_profiles WHERE user_id = 15440;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 555;\nSELECT * FROM crm_configurations WHERE id = 555;\nSELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 581 and sa.provider = 'salesforce';\n\nSELECT * FROM automated_report_results order by id desc;\n\nselect * from features;\nselect * from team_features where feature_id = 40;\n\nselect * from teams where id = 556;\n\nselect * from automated_reports;\nwhere id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , [\"pdf\",\"podcast\"]\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\nselect * from automated_report_results order by id desc;\nSELECT * FROM automated_report_results WHERE id = 1919;\n\nselect * from automated_report_results WHERE report_id = 54;\n\nselect * from opportunities where id = 7594349;\n\nSELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - jiminnyintegration@lesmills.com\nselect * from playbooks where team_id = 711; # event 226147\nSELECT * FROM playbook_categories WHERE playbook_id = 5515;\nSELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';\nSELECT * FROM crm_fields WHERE id = 226147;\nSELECT * FROM crm_field_values WHERE crm_field_id = 226147;\n\nSELECT * FROM crm_configurations WHERE id = 692;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 711 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;\n\nselect * from leads;\n\nselect * from calendars;\n\nSELECT\n t.id AS team_id,\n t.name,\n LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain\nFROM teams t\nJOIN users u ON u.team_id = t.id\nJOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'\nLEFT JOIN team_domains td\n ON td.team_id = t.id\n AND td.deleted_at IS NULL\n AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))\nGROUP BY t.id, t.name, calendar_domain\nORDER BY t.name, calendar_domain;\n\nselect * from users u join calendars c on c.user_id = u.id\nwhere u.team_id = 882;\n\n\nselect * from activities where id = 74049485; # team 563 crm 537\nselect * from activities where id = 73272382; # team 563 crm 537\nselect * from activities where id = 64400389; # team 563 crm 537\nselect * from activities where id = 58081273; # team 563 crm 537\nselect * from activities where id = 54520297; # team 563 crm 537\nselect * from participants where activity_id = 58081273;\n\nselect * from activities where crm_configuration_id = 537 and provider = 'aircall'\nand account_id = 19003658 order by updated_at desc;\n\nselect * from contacts where crm_configuration_id = 537 and id = 35957759;\nselect * from accounts where crm_configuration_id = 537 and id = 19003658;\n\nselect * from automated_report_results where id = 1976;\nselect * from automated_reports where id = 583;\nselect * from activity_searches where id = 87714;\nselect * from activity_search_filters where activity_search_id = 87714;\n\nSELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid\nor uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot';\nselect * from rate_limits;\n\nselect * from automated_report_results where media_type = 'pdf' and status = 2\nand id IN (18, 1872);\nselect * from automated_reports where id = 54;\nSELECT * FROM users WHERE id IN (24623,29443,29613);\n\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\n\nselect * from text_relays where created_at > '2026-01-01';\nand id IN (32415, 32416);\n# and id = 32412;\n\nselect * from users where team_id = 2 and email like '%scott%' and id = 29510;\n\nSELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436\n\nSELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses\nFROM text_relays\nWHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')\nGROUP BY email_provider_id;\nSELECT id, status, telephony_provider_id, created_at\nFROM activities\nWHERE id IN (80028719, 80028846);\nSELECT id, status, code, email_sent_at, created_at, updated_at\nFROM text_relays\nWHERE id IN (32415, 32416);\nSELECT id, status, code, sender, recipient, created_at\nFROM text_relays\nWHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'\nORDER BY created_at DESC\nLIMIT 10;\n\nSELECT id, uuid, status, code, sender, recipient, created_at, updated_at\nFROM text_relays\nWHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');\n\n# ***************\nSELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count\nFROM users u\nINNER JOIN activities a ON u.id = a.user_id\nWHERE a.type LIKE 'sms%'\nAND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)\nGROUP BY u.id, u.email, u.name, u.softphone_number\nORDER BY sms_count DESC;\n\nselect * from teams where id = 1;\n\nselect * from roles;","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Project","depth":3,"on_screen":false,"role_description":"text"},{"role":"AXButton","text":"Project","depth":3,"bounds":{"left":0.011968086,"top":0.047885075,"width":0.024268618,"height":0.024740623},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"New File or Directory…","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Expand Selected","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Collapse All","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Options","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false}]...
|
6081160626368098919
|
2218652917440067143
|
visual_change
|
accessibility
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993
SELECT * FROM users WHERE id = 25061;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 994;
SELECT * FROM crm_profiles WHERE user_id = 25061;
select * from crm_configurations where id = 834;
SELECT * FROM teams WHERE id = 882;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;
SELECT * FROM contacts where crm_configuration_id = 834;
SELECT * FROM opportunities WHERE team_id = 933
# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');
AND id IN (8482561,18352941,19042734,19232139,19445140,19472541);
SELECT * FROM opportunity_contacts
WHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 485; #
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
select crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id
where crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')
# and l.converted_at IS NOT NULL
;
# [PASSWORD_DOTS]
SELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')
and opportunity_id IS NULL
order by id desc;
SELECT * FROM teams WHERE id = 604; # 598
SELECT * FROM activities WHERE id = 74410828; # [EMAIL]
SELECT * FROM accounts WHERE id = 20068382;
SELECT * FROM accounts WHERE id = 35186038;
SELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 559 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;
select * from sidekick_settings where team_id = 781;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100
SELECT * FROM crm_layouts WHERE crm_configuration_id = 711;
SELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL
and is_internal = 0 and status = 'completed'
order by id desc;
SELECT * FROM crm_layout_entities
WHERE crm_layout_id IN (2352, 2353);
;
SELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 556 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;
SELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;
select * from contacts
where crm_configuration_id = 530
and crm_provider_id = 872252;
select * from activities where crm_configuration_id = 530
and user_id = 14343 and type like '%softphone%'
and created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);
SELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t
JOIN crm_configurations c ON t.id = c.team_id
WHERE t.status = 'active';
SELECT * FROM teams where id = 1091;
SELECT * FROM crm_configurations where team_id = 1091;
SELECT * FROM activity_providers where team_id = 1091;
SELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT * FROM teams WHERE name LIKE '%Leadventure%';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1091 and sa.provider = 'salesforce';
SELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812
SELECT * FROM teams where id = 862;
SELECT * FROM crm_configurations where team_id = 862;
SELECT * FROM activity_providers where team_id = 862;
SELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT t.id, crm.id, crm.provider, ap.* FROM teams t
join crm_configurations crm on t.id = crm.team_id
join activity_providers ap on t.id = ap.team_id
where t.status = 'active' and ap.is_enabled = 1
and crm.provider = 'hubspot'
and ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',
'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');
SELECT * FROM teams where id = 1068;
SELECT * FROM crm_configurations where team_id = 1068;
SELECT * FROM activity_providers where team_id = 1068;
SELECT * FROM activities a
where crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')
and a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'
)
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by a.id desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1068 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262
SELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
select * from crm_layouts where crm_configuration_id = 834;
select * from crm_layout_entities where crm_layout_id = 2780;
select * from crm_fields where id IN (321153,321192,321193,321194);
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1057 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8
SELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20
SELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10
SELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #
SELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;
select * from users where team_id = 51; # 7783
SELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130
select * from activity_searches where user_id = 7783;
select * from activity_search_filters where activity_search_id IN (32291, 32292);
SELECT asf.activity_search_id, asf.id, asf.value
FROM activity_search_filters asf
WHERE asf.filter = 'group_id'
AND asf.value IN (
SELECT CONCAT(
HEX(SUBSTR(uuid, 5, 4)), '-',
HEX(SUBSTR(uuid, 3, 2)), '-',
HEX(SUBSTR(uuid, 1, 2)), '-',
HEX(SUBSTR(uuid, 9, 2)), '-',
HEX(SUBSTR(uuid, 11))
)
FROM groups
WHERE deleted_at IS NOT NULL
);
SELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where provider = 'hubspot';
SELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133
SELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null
# [PASSWORD_DOTS]
select * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';
select
cp.*
# DISTINCT t.id
# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields
FROM crm_profiles cp
JOIN crm_configurations crm on crm.id = cp.crm_configuration_id
JOIN users u on u.id = cp.user_id
JOIN teams t ON t.id = crm.team_id
WHERE crm.provider = 'salesforce' and t.status = 'active'
and cp.archived_at IS NULL and u.deleted_at IS NULL
and t.id NOT IN (1093)
and t.id = 2
and cp.contact_fields IS NULL;
# and c.crm_provider_id = '003Uu00000ojD4NIAU';
SELECT * FROM users WHERE id = 26484;
SELECT * FROM crm_profiles WHERE user_id = 26484;
SELECT * FROM social_accounts WHERE sociable_id = 26484;
SELECT * FROM crm_configurations where provider = 'salesforce';
select * from users where id IN (10022, 10403);
select * from users where team_id IN (526);
select * from teams where id IN (526, 532);
select * from crm_configurations where id IN (500, 516);
select * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);
select * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 526 and sa.provider = 'salesforce';
select * from team_settings where team_id IN (526, 532);
select * from users where id IN (22824);
select * from crm_profiles where crm_configuration_id IN (1026);
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1093 and sa.provider = 'salesforce';
select * from teams where id = 1099;
select * from users where id = 29643
select * from activity_processing_states;
SELECT * FROM teams where name LIKE '%Fare%'; # 233
SELECT * FROM opportunities where crm_configuration_id = 215
# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'
;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1088 and sa.provider = 'hubspot';
SELECT * FROM teams order by updated_at DESC
SELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account
select * from crm_configurations where provider = 'pipedrive';
select * from teams where id = 957;
select * from crm_configurations where id = 957;
SELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743
SELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;
select * from users where team_id = 1; # 26726 - Gabriela Dureva
SELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific
select * from activities where user_id = 26726 order by id desc;
select * from contacts where crm_configuration_id = 1
and email IN ('[EMAIL]', '[EMAIL]'); # 2094416, 2093620
SELECT * FROM contacts WHERE id = 6284931;
SELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id
WHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;
select * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);
select * from crm_configurations where id = 1;
43801692-1aeb-32ce-acba-5b80a479701a
44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b
405975c0-b3d0-7aaa-821f-09d59cae6dd1
4caf848d-4bed-2299-b248-7788d41f9fca
49bedc3f-f196-eef3-89c3-dea6a3b4aa63
43420989-a09d-b8f8-9806-c8bbf7a02aac
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
SELECT * FROM activities WHERE id = 75461988;
SELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;
select * from contacts where id = 17900517;
select * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id
where crm.provider != 'salesforce';
select * from users where id = 21047;
SELECT * FROM crm_configurations WHERE id = 892;
SELECT * FROM teams WHERE id = 942;
select * from opportunities where team_id = 942 order by updated_at desc;
select * from contacts where team_id = 942 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 942 and sa.provider = 'hubspot';
SELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430
SELECT * FROM crm_configurations WHERE id = 1;
SELECT * FROM teams WHERE crm_id = 1;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
select id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1
SELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430
select * from teams where id = 852;
select * from groups where id = 2286;
select * from sidekick_settings where team_id = 852;
select * from default_activity_types where team_id = 852;
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1 AND u.deleted_at IS NULL
AND u.crm_required = 1
AND u.team_id = 1
ORDER BY u.team_id;
SELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (
18481
);
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1
AND u.deleted_at IS NULL
AND u.crm_required = 1
# AND u.team_id = 1
AND p.id IS NULL -- Move this condition to WHERE clause
ORDER BY u.team_id;
SELECT * FROM opportunities WHERE id = 20002609;
select * from teams where id = 1122; # Velatir, 29953 - [EMAIL]
select * from crm_configurations where id = 1060;
select * from crm_layouts where crm_configuration_id = 1060;
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1122 and sa.provider = 'hubspot';
select * from opportunities where team_id = 1122 order by updated_at desc;
select * from crm_field_data where object_type = 'contact';
SELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 248 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS
SELECT * FROM users where id = 24115;
SELECT * FROM accounts where id = 4002896;
SELECT * FROM teams WHERE name LIKE '%adswerve%';
SELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN ("0069N000003GIQ9QAO","0061r000019yGP9AAM","0066900001S2KWlAAN","0066900001TDpj2AAD","0066900001b8uEwAAI","0069N000001rQi0QAE","006QF00000KD40mYAD","006QF00000LzpRJYAZ","0069N000002uomtQAA","0069N000002xlMLQAY","0066900001NV6ubAAD","0061r00001HJp45AAD","006QF00000uTlUoYAK","006QF00000v0bZqYAI");
SELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203
SELECT u.id, u.email, ac.name, a.* FROM activities a
JOIN users u ON a.user_id = u.id
JOIN accounts ac ON a.account_id = ac.id
WHERE
uuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or
uuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or
uuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;
select * from users where id = 5825;
SELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;
select * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;
19594, 862
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 862 and sa.provider = 'salesforce';
select * from automated_reports where id = 36;
select ar.frequency, r.*, ar.* from automated_report_results r
join automated_reports ar on r.report_id = ar.id
where ar.frequency != 'one_off';
select s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;
select * from nudges n where n.activity_search_id
select * from teams where created_at > '2026-03-09';
SELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;
select * from users where team_id = 1 and name like '%Lukas%'; # 7160
SELECT * FROM teams WHERE id = 575;
select * from opportunities where team_id = 575;
SELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,
select * from opportunities where team_id = 1126;
SELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,
select * from opportunities where team_id = 1125;
select * from contacts c
where c.team_id = 882;
SELECT * FROM activities WHERE id = 76822967;
SELECT * FROM crm_profiles WHERE user_id = 15440;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 555;
SELECT * FROM crm_configurations WHERE id = 555;
SELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 581 and sa.provider = 'salesforce';
SELECT * FROM automated_report_results order by id desc;
select * from features;
select * from team_features where feature_id = 40;
select * from teams where id = 556;
select * from automated_reports;
where id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , ["pdf","podcast"]
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from automated_report_results order by id desc;
SELECT * FROM automated_report_results WHERE id = 1919;
select * from automated_report_results WHERE report_id = 54;
select * from opportunities where id = 7594349;
SELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - [EMAIL]
select * from playbooks where team_id = 711; # event 226147
SELECT * FROM playbook_categories WHERE playbook_id = 5515;
SELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';
SELECT * FROM crm_fields WHERE id = 226147;
SELECT * FROM crm_field_values WHERE crm_field_id = 226147;
SELECT * FROM crm_configurations WHERE id = 692;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 711 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;
select * from leads;
select * from calendars;
SELECT
t.id AS team_id,
t.name,
LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain
FROM teams t
JOIN users u ON u.team_id = t.id
JOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'
LEFT JOIN team_domains td
ON td.team_id = t.id
AND td.deleted_at IS NULL
AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))
GROUP BY t.id, t.name, calendar_domain
ORDER BY t.name, calendar_domain;
select * from users u join calendars c on c.user_id = u.id
where u.team_id = 882;
select * from activities where id = 74049485; # team 563 crm 537
select * from activities where id = 73272382; # team 563 crm 537
select * from activities where id = 64400389; # team 563 crm 537
select * from activities where id = 58081273; # team 563 crm 537
select * from activities where id = 54520297; # team 563 crm 537
select * from participants where activity_id = 58081273;
select * from activities where crm_configuration_id = 537 and provider = 'aircall'
and account_id = 19003658 order by updated_at desc;
select * from contacts where crm_configuration_id = 537 and id = 35957759;
select * from accounts where crm_configuration_id = 537 and id = 19003658;
select * from automated_report_results where id = 1976;
select * from automated_reports where id = 583;
select * from activity_searches where id = 87714;
select * from activity_search_filters where activity_search_id = 87714;
SELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid
or uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;
SELECT * FROM crm_configurations WHERE provider = 'hubspot';
select * from rate_limits;
select * from automated_report_results where media_type = 'pdf' and status = 2
and id IN (18, 1872);
select * from automated_reports where id = 54;
SELECT * FROM users WHERE id IN (24623,29443,29613);
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from text_relays where created_at > '2026-01-01';
and id IN (32415, 32416);
# and id = 32412;
select * from users where team_id = 2 and email like '%scott%' and id = 29510;
SELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436
SELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses
FROM text_relays
WHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')
GROUP BY email_provider_id;
SELECT id, status, telephony_provider_id, created_at
FROM activities
WHERE id IN (80028719, 80028846);
SELECT id, status, code, email_sent_at, created_at, updated_at
FROM text_relays
WHERE id IN (32415, 32416);
SELECT id, status, code, sender, recipient, created_at
FROM text_relays
WHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'
ORDER BY created_at DESC
LIMIT 10;
SELECT id, uuid, status, code, sender, recipient, created_at, updated_at
FROM text_relays
WHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');
# [PASSWORD_DOTS]
SELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count
FROM users u
INNER JOIN activities a ON u.id = a.user_id
WHERE a.type LIKE 'sms%'
AND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)
GROUP BY u.id, u.email, u.name, u.softphone_number
ORDER BY sms_count DESC;
select * from teams where id = 1;
select * from roles;
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
64668
|
NULL
|
NULL
|
NULL
|
|
64669
|
2264
|
23
|
2026-05-20T14:11:05.235343+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286265235_m1.jpg...
|
PhpStorm
|
faVsco.js – console [PROD]
|
1
|
NULL
|
monitor_1
|
NULL
|
NULL
|
NULL
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993
SELECT * FROM users WHERE id = 25061;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 994;
SELECT * FROM crm_profiles WHERE user_id = 25061;
select * from crm_configurations where id = 834;
SELECT * FROM teams WHERE id = 882;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;
SELECT * FROM contacts where crm_configuration_id = 834;
SELECT * FROM opportunities WHERE team_id = 933
# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');
AND id IN (8482561,18352941,19042734,19232139,19445140,19472541);
SELECT * FROM opportunity_contacts
WHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 485; #
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
select crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id
where crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')
# and l.converted_at IS NOT NULL
;
# [PASSWORD_DOTS]
SELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')
and opportunity_id IS NULL
order by id desc;
SELECT * FROM teams WHERE id = 604; # 598
SELECT * FROM activities WHERE id = 74410828; # [EMAIL]
SELECT * FROM accounts WHERE id = 20068382;
SELECT * FROM accounts WHERE id = 35186038;
SELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 559 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;
select * from sidekick_settings where team_id = 781;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100
SELECT * FROM crm_layouts WHERE crm_configuration_id = 711;
SELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL
and is_internal = 0 and status = 'completed'
order by id desc;
SELECT * FROM crm_layout_entities
WHERE crm_layout_id IN (2352, 2353);
;
SELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 556 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;
SELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;
select * from contacts
where crm_configuration_id = 530
and crm_provider_id = 872252;
select * from activities where crm_configuration_id = 530
and user_id = 14343 and type like '%softphone%'
and created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);
SELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t
JOIN crm_configurations c ON t.id = c.team_id
WHERE t.status = 'active';
SELECT * FROM teams where id = 1091;
SELECT * FROM crm_configurations where team_id = 1091;
SELECT * FROM activity_providers where team_id = 1091;
SELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT * FROM teams WHERE name LIKE '%Leadventure%';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1091 and sa.provider = 'salesforce';
SELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812
SELECT * FROM teams where id = 862;
SELECT * FROM crm_configurations where team_id = 862;
SELECT * FROM activity_providers where team_id = 862;
SELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT t.id, crm.id, crm.provider, ap.* FROM teams t
join crm_configurations crm on t.id = crm.team_id
join activity_providers ap on t.id = ap.team_id
where t.status = 'active' and ap.is_enabled = 1
and crm.provider = 'hubspot'
and ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',
'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');
SELECT * FROM teams where id = 1068;
SELECT * FROM crm_configurations where team_id = 1068;
SELECT * FROM activity_providers where team_id = 1068;
SELECT * FROM activities a
where crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')
and a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'
)
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by a.id desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1068 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262
SELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
select * from crm_layouts where crm_configuration_id = 834;
select * from crm_layout_entities where crm_layout_id = 2780;
select * from crm_fields where id IN (321153,321192,321193,321194);
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1057 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8
SELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20
SELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10
SELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #
SELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;
select * from users where team_id = 51; # 7783
SELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130
select * from activity_searches where user_id = 7783;
select * from activity_search_filters where activity_search_id IN (32291, 32292);
SELECT asf.activity_search_id, asf.id, asf.value
FROM activity_search_filters asf
WHERE asf.filter = 'group_id'
AND asf.value IN (
SELECT CONCAT(
HEX(SUBSTR(uuid, 5, 4)), '-',
HEX(SUBSTR(uuid, 3, 2)), '-',
HEX(SUBSTR(uuid, 1, 2)), '-',
HEX(SUBSTR(uuid, 9, 2)), '-',
HEX(SUBSTR(uuid, 11))
)
FROM groups
WHERE deleted_at IS NOT NULL
);
SELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where provider = 'hubspot';
SELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133
SELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null
# [PASSWORD_DOTS]
select * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';
select
cp.*
# DISTINCT t.id
# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields
FROM crm_profiles cp
JOIN crm_configurations crm on crm.id = cp.crm_configuration_id
JOIN users u on u.id = cp.user_id
JOIN teams t ON t.id = crm.team_id
WHERE crm.provider = 'salesforce' and t.status = 'active'
and cp.archived_at IS NULL and u.deleted_at IS NULL
and t.id NOT IN (1093)
and t.id = 2
and cp.contact_fields IS NULL;
# and c.crm_provider_id = '003Uu00000ojD4NIAU';
SELECT * FROM users WHERE id = 26484;
SELECT * FROM crm_profiles WHERE user_id = 26484;
SELECT * FROM social_accounts WHERE sociable_id = 26484;
SELECT * FROM crm_configurations where provider = 'salesforce';
select * from users where id IN (10022, 10403);
select * from users where team_id IN (526);
select * from teams where id IN (526, 532);
select * from crm_configurations where id IN (500, 516);
select * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);
select * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 526 and sa.provider = 'salesforce';
select * from team_settings where team_id IN (526, 532);
select * from users where id IN (22824);
select * from crm_profiles where crm_configuration_id IN (1026);
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1093 and sa.provider = 'salesforce';
select * from teams where id = 1099;
select * from users where id = 29643
select * from activity_processing_states;
SELECT * FROM teams where name LIKE '%Fare%'; # 233
SELECT * FROM opportunities where crm_configuration_id = 215
# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'
;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1088 and sa.provider = 'hubspot';
SELECT * FROM teams order by updated_at DESC
SELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account
select * from crm_configurations where provider = 'pipedrive';
select * from teams where id = 957;
select * from crm_configurations where id = 957;
SELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743
SELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;
select * from users where team_id = 1; # 26726 - Gabriela Dureva
SELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific
select * from activities where user_id = 26726 order by id desc;
select * from contacts where crm_configuration_id = 1
and email IN ('[EMAIL]', '[EMAIL]'); # 2094416, 2093620
SELECT * FROM contacts WHERE id = 6284931;
SELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id
WHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;
select * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);
select * from crm_configurations where id = 1;
43801692-1aeb-32ce-acba-5b80a479701a
44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b
405975c0-b3d0-7aaa-821f-09d59cae6dd1
4caf848d-4bed-2299-b248-7788d41f9fca
49bedc3f-f196-eef3-89c3-dea6a3b4aa63
43420989-a09d-b8f8-9806-c8bbf7a02aac
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
SELECT * FROM activities WHERE id = 75461988;
SELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;
select * from contacts where id = 17900517;
select * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id
where crm.provider != 'salesforce';
select * from users where id = 21047;
SELECT * FROM crm_configurations WHERE id = 892;
SELECT * FROM teams WHERE id = 942;
select * from opportunities where team_id = 942 order by updated_at desc;
select * from contacts where team_id = 942 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 942 and sa.provider = 'hubspot';
SELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430
SELECT * FROM crm_configurations WHERE id = 1;
SELECT * FROM teams WHERE crm_id = 1;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
select id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1
SELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430
select * from teams where id = 852;
select * from groups where id = 2286;
select * from sidekick_settings where team_id = 852;
select * from default_activity_types where team_id = 852;
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1 AND u.deleted_at IS NULL
AND u.crm_required = 1
AND u.team_id = 1
ORDER BY u.team_id;
SELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (
18481
);
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1
AND u.deleted_at IS NULL
AND u.crm_required = 1
# AND u.team_id = 1
AND p.id IS NULL -- Move this condition to WHERE clause
ORDER BY u.team_id;
SELECT * FROM opportunities WHERE id = 20002609;
select * from teams where id = 1122; # Velatir, 29953 - [EMAIL]
select * from crm_configurations where id = 1060;
select * from crm_layouts where crm_configuration_id = 1060;
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1122 and sa.provider = 'hubspot';
select * from opportunities where team_id = 1122 order by updated_at desc;
select * from crm_field_data where object_type = 'contact';
SELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 248 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS
SELECT * FROM users where id = 24115;
SELECT * FROM accounts where id = 4002896;
SELECT * FROM teams WHERE name LIKE '%adswerve%';
SELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN ("0069N000003GIQ9QAO","0061r000019yGP9AAM","0066900001S2KWlAAN","0066900001TDpj2AAD","0066900001b8uEwAAI","0069N000001rQi0QAE","006QF00000KD40mYAD","006QF00000LzpRJYAZ","0069N000002uomtQAA","0069N000002xlMLQAY","0066900001NV6ubAAD","0061r00001HJp45AAD","006QF00000uTlUoYAK","006QF00000v0bZqYAI");
SELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203
SELECT u.id, u.email, ac.name, a.* FROM activities a
JOIN users u ON a.user_id = u.id
JOIN accounts ac ON a.account_id = ac.id
WHERE
uuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or
uuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or
uuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;
select * from users where id = 5825;
SELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;
select * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;
19594, 862
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 862 and sa.provider = 'salesforce';
select * from automated_reports where id = 36;
select ar.frequency, r.*, ar.* from automated_report_results r
join automated_reports ar on r.report_id = ar.id
where ar.frequency != 'one_off';
select s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;
select * from nudges n where n.activity_search_id
select * from teams where created_at > '2026-03-09';
SELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;
select * from users where team_id = 1 and name like '%Lukas%'; # 7160
SELECT * FROM teams WHERE id = 575;
select * from opportunities where team_id = 575;
SELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,
select * from opportunities where team_id = 1126;
SELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,
select * from opportunities where team_id = 1125;
select * from contacts c
where c.team_id = 882;
SELECT * FROM activities WHERE id = 76822967;
SELECT * FROM crm_profiles WHERE user_id = 15440;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 555;
SELECT * FROM crm_configurations WHERE id = 555;
SELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 581 and sa.provider = 'salesforce';
SELECT * FROM automated_report_results order by id desc;
select * from features;
select * from team_features where feature_id = 40;
select * from teams where id = 556;
select * from automated_reports;
where id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , ["pdf","podcast"]
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from automated_report_results order by id desc;
SELECT * FROM automated_report_results WHERE id = 1919;
select * from automated_report_results WHERE report_id = 54;
select * from opportunities where id = 7594349;
SELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - [EMAIL]
select * from playbooks where team_id = 711; # event 226147
SELECT * FROM playbook_categories WHERE playbook_id = 5515;
SELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';
SELECT * FROM crm_fields WHERE id = 226147;
SELECT * FROM crm_field_values WHERE crm_field_id = 226147;
SELECT * FROM crm_configurations WHERE id = 692;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 711 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;
select * from leads;
select * from calendars;
SELECT
t.id AS team_id,
t.name,
LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain
FROM teams t
JOIN users u ON u.team_id = t.id
JOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'
LEFT JOIN team_domains td
ON td.team_id = t.id
AND td.deleted_at IS NULL
AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))
GROUP BY t.id, t.name, calendar_domain
ORDER BY t.name, calendar_domain;
select * from users u join calendars c on c.user_id = u.id
where u.team_id = 882;
select * from activities where id = 74049485; # team 563 crm 537
select * from activities where id = 73272382; # team 563 crm 537
select * from activities where id = 64400389; # team 563 crm 537
select * from activities where id = 58081273; # team 563 crm 537
select * from activities where id = 54520297; # team 563 crm 537
select * from participants where activity_id = 58081273;
select * from activities where crm_configuration_id = 537 and provider = 'aircall'
and account_id = 19003658 order by updated_at desc;
select * from contacts where crm_configuration_id = 537 and id = 35957759;
select * from accounts where crm_configuration_id = 537 and id = 19003658;
select * from automated_report_results where id = 1976;
select * from automated_reports where id = 583;
select * from activity_searches where id = 87714;
select * from activity_search_filters where activity_search_id = 87714;
SELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid
or uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;
SELECT * FROM crm_configurations WHERE provider = 'hubspot';
select * from rate_limits;
select * from automated_report_results where media_type = 'pdf' and status = 2
and id IN (18, 1872);
select * from automated_reports where id = 54;
SELECT * FROM users WHERE id IN (24623,29443,29613);
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from text_relays where created_at > '2026-01-01';
and id IN (32415, 32416);
# and id = 32412;
select * from users where team_id = 2 and email like '%scott%' and id = 29510;
SELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436
SELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses
FROM text_relays
WHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')
GROUP BY email_provider_id;
SELECT id, status, telephony_provider_id, created_at
FROM activities
WHERE id IN (80028719, 80028846);
SELECT id, status, code, email_sent_at, created_at, updated_at
FROM text_relays
WHERE id IN (32415, 32416);
SELECT id, status, code, sender, recipient, created_at
FROM text_relays
WHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'
ORDER BY created_at DESC
LIMIT 10;
SELECT id, uuid, status, code, sender, recipient, created_at, updated_at
FROM text_relays
WHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');
# [PASSWORD_DOTS]
SELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count
FROM users u
INNER JOIN activities a ON u.id = a.user_id
WHERE a.type LIKE 'sms%'
AND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)
GROUP BY u.id, u.email, u.name, u.softphone_number
ORDER BY sms_count DESC;
select * from teams where id = 1;
select * from roles;
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
[{"role":"AXButton","text" [{"role":"AXButton","text":"Project: faVsco.js, menu","depth":5,"on_screen":true,"help_text":"~/jiminny/app","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"#12105 on JY-20915-add-domain-specific-emai…relay, menu","depth":5,"on_screen":true,"help_text":"Pull request #12105 exists for current branch JY-20915-add-domain-specific-emai…relay","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Start Listening for PHP Debug Connections","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"SmsReceivedTest","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Run 'SmsReceivedTest'","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Debug 'SmsReceivedTest'","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"More Actions","depth":6,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"JetBrains AI","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Search Everywhere","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"IDE and Project Settings","depth":5,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.088194445,"height":0.027777778},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"3","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"3","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","depth":4,"on_screen":true,"value":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Execute","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Explain Plan","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Browse Query History","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"View Parameters","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Open Query Execution Settings…","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"In-Editor Results","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Tx: Auto","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Cancel Running Statements","depth":4,"on_screen":true,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Playground","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"jiminny","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.088194445,"height":0.027777778},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"41","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"1","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"40","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"65","depth":4,"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993\nSELECT * FROM users WHERE id = 25061;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 994;\nSELECT * FROM crm_profiles WHERE user_id = 25061;\n\nselect * from crm_configurations where id = 834;\nSELECT * FROM teams WHERE id = 882;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;\n\nSELECT * FROM contacts where crm_configuration_id = 834;\nSELECT * FROM opportunities WHERE team_id = 933\n# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');\nAND id IN (8482561,18352941,19042734,19232139,19445140,19472541);\nSELECT * FROM opportunity_contacts\nWHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 485; #\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nselect crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id\nwhere crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')\n# and l.converted_at IS NOT NULL\n;\n\n# ********************************************************************\nSELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')\nand opportunity_id IS NULL\norder by id desc;\n\nSELECT * FROM teams WHERE id = 604; # 598\nSELECT * FROM activities WHERE id = 74410828; # chelseaw@allvoices.co\nSELECT * FROM accounts WHERE id = 20068382;\nSELECT * FROM accounts WHERE id = 35186038;\n\nSELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 559 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;\nselect * from sidekick_settings where team_id = 781;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100\n\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 711;\nSELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL\nand is_internal = 0 and status = 'completed'\norder by id desc;\n\nSELECT * FROM crm_layout_entities\nWHERE crm_layout_id IN (2352, 2353);\n;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 556 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;\nSELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;\nselect * from contacts\nwhere crm_configuration_id = 530\nand crm_provider_id = 872252;\n\nselect * from activities where crm_configuration_id = 530\nand user_id = 14343 and type like '%softphone%'\nand created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);\n\n\nSELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t\nJOIN crm_configurations c ON t.id = c.team_id\nWHERE t.status = 'active';\n\nSELECT * FROM teams where id = 1091;\nSELECT * FROM crm_configurations where team_id = 1091;\nSELECT * FROM activity_providers where team_id = 1091;\nSELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT * FROM teams WHERE name LIKE '%Leadventure%';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1091 and sa.provider = 'salesforce';\n\nSELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812\nSELECT * FROM teams where id = 862;\nSELECT * FROM crm_configurations where team_id = 862;\nSELECT * FROM activity_providers where team_id = 862;\nSELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT t.id, crm.id, crm.provider, ap.* FROM teams t\njoin crm_configurations crm on t.id = crm.team_id\njoin activity_providers ap on t.id = ap.team_id\nwhere t.status = 'active' and ap.is_enabled = 1\nand crm.provider = 'hubspot'\nand ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',\n 'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');\n\nSELECT * FROM teams where id = 1068;\nSELECT * FROM crm_configurations where team_id = 1068;\nSELECT * FROM activity_providers where team_id = 1068;\n\nSELECT * FROM activities a\nwhere crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')\nand a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'\n )\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by a.id desc;\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1068 and sa.provider = 'hubspot';\n\n# ********************************************************************\n# ********************************************************************\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262\nSELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\nselect * from crm_layouts where crm_configuration_id = 834;\nselect * from crm_layout_entities where crm_layout_id = 2780;\nselect * from crm_fields where id IN (321153,321192,321193,321194);\n\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1057 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8\n\nSELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20\n\nSELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10\n\nSELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #\n\nSELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;\nselect * from users where team_id = 51; # 7783\nSELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130\nselect * from activity_searches where user_id = 7783;\nselect * from activity_search_filters where activity_search_id IN (32291, 32292);\n\nSELECT asf.activity_search_id, asf.id, asf.value\nFROM activity_search_filters asf\nWHERE asf.filter = 'group_id'\nAND asf.value IN (\n SELECT CONCAT(\n HEX(SUBSTR(uuid, 5, 4)), '-',\n HEX(SUBSTR(uuid, 3, 2)), '-',\n HEX(SUBSTR(uuid, 1, 2)), '-',\n HEX(SUBSTR(uuid, 9, 2)), '-',\n HEX(SUBSTR(uuid, 11))\n )\n FROM groups\n WHERE deleted_at IS NOT NULL\n);\n\nSELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th\n# ********************************************************************\nSELECT * FROM crm_configurations where provider = 'hubspot';\nSELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133\nSELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null\n# ********************************************************************\n\nselect * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';\nselect\n cp.*\n# DISTINCT t.id\n# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields\nFROM crm_profiles cp\nJOIN crm_configurations crm on crm.id = cp.crm_configuration_id\nJOIN users u on u.id = cp.user_id\nJOIN teams t ON t.id = crm.team_id\nWHERE crm.provider = 'salesforce' and t.status = 'active'\n and cp.archived_at IS NULL and u.deleted_at IS NULL\n and t.id NOT IN (1093)\n and t.id = 2\n and cp.contact_fields IS NULL;\n# and c.crm_provider_id = '003Uu00000ojD4NIAU';\n\nSELECT * FROM users WHERE id = 26484;\nSELECT * FROM crm_profiles WHERE user_id = 26484;\nSELECT * FROM social_accounts WHERE sociable_id = 26484;\nSELECT * FROM crm_configurations where provider = 'salesforce';\nselect * from users where id IN (10022, 10403);\nselect * from users where team_id IN (526);\nselect * from teams where id IN (526, 532);\nselect * from crm_configurations where id IN (500, 516);\nselect * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);\nselect * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 526 and sa.provider = 'salesforce';\nselect * from team_settings where team_id IN (526, 532);\n\nselect * from users where id IN (22824);\nselect * from crm_profiles where crm_configuration_id IN (1026);\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1093 and sa.provider = 'salesforce';\n\nselect * from teams where id = 1099;\nselect * from users where id = 29643\n\nselect * from activity_processing_states;\n\nSELECT * FROM teams where name LIKE '%Fare%'; # 233\nSELECT * FROM opportunities where crm_configuration_id = 215\n# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'\n;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1088 and sa.provider = 'hubspot';\n\nSELECT * FROM teams order by updated_at DESC\nSELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account\n\nselect * from crm_configurations where provider = 'pipedrive';\n\nselect * from teams where id = 957;\nselect * from crm_configurations where id = 957;\n\nSELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743\nSELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;\n\nselect * from users where team_id = 1; # 26726 - Gabriela Dureva\nSELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific\nselect * from activities where user_id = 26726 order by id desc;\nselect * from contacts where crm_configuration_id = 1\nand email IN ('charlotte.ward@prolific.com', 'frankie.bryant@prolific.com'); # 2094416, 2093620\nSELECT * FROM contacts WHERE id = 6284931;\n\nSELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id\nWHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;\n\nselect * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);\nselect * from crm_configurations where id = 1;\n\n43801692-1aeb-32ce-acba-5b80a479701a\n44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b\n405975c0-b3d0-7aaa-821f-09d59cae6dd1\n4caf848d-4bed-2299-b248-7788d41f9fca\n49bedc3f-f196-eef3-89c3-dea6a3b4aa63\n43420989-a09d-b8f8-9806-c8bbf7a02aac\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nSELECT * FROM activities WHERE id = 75461988;\n\nSELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;\n\nselect * from contacts where id = 17900517;\n\nselect * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id\nwhere crm.provider != 'salesforce';\n\nselect * from users where id = 21047;\nSELECT * FROM crm_configurations WHERE id = 892;\nSELECT * FROM teams WHERE id = 942;\nselect * from opportunities where team_id = 942 order by updated_at desc;\nselect * from contacts where team_id = 942 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 942 and sa.provider = 'hubspot';\n\nSELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430\nSELECT * FROM crm_configurations WHERE id = 1;\nSELECT * FROM teams WHERE crm_id = 1;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nselect id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1\nSELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430\n\nselect * from teams where id = 852;\nselect * from groups where id = 2286;\nselect * from sidekick_settings where team_id = 852;\nselect * from default_activity_types where team_id = 852;\n\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1 AND u.deleted_at IS NULL\nAND u.crm_required = 1\nAND u.team_id = 1\nORDER BY u.team_id;\n\nSELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (\n18481\n );\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1\n AND u.deleted_at IS NULL\n AND u.crm_required = 1\n# AND u.team_id = 1\n AND p.id IS NULL -- Move this condition to WHERE clause\nORDER BY u.team_id;\n\nSELECT * FROM opportunities WHERE id = 20002609;\nselect * from teams where id = 1122; # Velatir, 29953 - christian@velatir.com\nselect * from crm_configurations where id = 1060;\nselect * from crm_layouts where crm_configuration_id = 1060;\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1122 and sa.provider = 'hubspot';\nselect * from opportunities where team_id = 1122 order by updated_at desc;\n\nselect * from crm_field_data where object_type = 'contact';\n\nSELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 248 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS\nSELECT * FROM users where id = 24115;\nSELECT * FROM accounts where id = 4002896;\nSELECT * FROM teams WHERE name LIKE '%adswerve%';\nSELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN (\"0069N000003GIQ9QAO\",\"0061r000019yGP9AAM\",\"0066900001S2KWlAAN\",\"0066900001TDpj2AAD\",\"0066900001b8uEwAAI\",\"0069N000001rQi0QAE\",\"006QF00000KD40mYAD\",\"006QF00000LzpRJYAZ\",\"0069N000002uomtQAA\",\"0069N000002xlMLQAY\",\"0066900001NV6ubAAD\",\"0061r00001HJp45AAD\",\"006QF00000uTlUoYAK\",\"006QF00000v0bZqYAI\");\nSELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203\n\nSELECT u.id, u.email, ac.name, a.* FROM activities a\nJOIN users u ON a.user_id = u.id\nJOIN accounts ac ON a.account_id = ac.id\nWHERE\nuuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or\nuuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or\nuuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;\n\nselect * from users where id = 5825;\nSELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;\n\nselect * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;\n19594, 862\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 862 and sa.provider = 'salesforce';\n\nselect * from automated_reports where id = 36;\nselect ar.frequency, r.*, ar.* from automated_report_results r\njoin automated_reports ar on r.report_id = ar.id\nwhere ar.frequency != 'one_off';\n\nselect s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;\nselect * from nudges n where n.activity_search_id\n\nselect * from teams where created_at > '2026-03-09';\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;\n\nselect * from users where team_id = 1 and name like '%Lukas%'; # 7160\n\nSELECT * FROM teams WHERE id = 575;\nselect * from opportunities where team_id = 575;\nSELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,\nselect * from opportunities where team_id = 1126;\nSELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,\nselect * from opportunities where team_id = 1125;\nselect * from contacts c\nwhere c.team_id = 882;\n\nSELECT * FROM activities WHERE id = 76822967;\nSELECT * FROM crm_profiles WHERE user_id = 15440;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 555;\nSELECT * FROM crm_configurations WHERE id = 555;\nSELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 581 and sa.provider = 'salesforce';\n\nSELECT * FROM automated_report_results order by id desc;\n\nselect * from features;\nselect * from team_features where feature_id = 40;\n\nselect * from teams where id = 556;\n\nselect * from automated_reports;\nwhere id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , [\"pdf\",\"podcast\"]\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\nselect * from automated_report_results order by id desc;\nSELECT * FROM automated_report_results WHERE id = 1919;\n\nselect * from automated_report_results WHERE report_id = 54;\n\nselect * from opportunities where id = 7594349;\n\nSELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - jiminnyintegration@lesmills.com\nselect * from playbooks where team_id = 711; # event 226147\nSELECT * FROM playbook_categories WHERE playbook_id = 5515;\nSELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';\nSELECT * FROM crm_fields WHERE id = 226147;\nSELECT * FROM crm_field_values WHERE crm_field_id = 226147;\n\nSELECT * FROM crm_configurations WHERE id = 692;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 711 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;\n\nselect * from leads;\n\nselect * from calendars;\n\nSELECT\n t.id AS team_id,\n t.name,\n LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain\nFROM teams t\nJOIN users u ON u.team_id = t.id\nJOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'\nLEFT JOIN team_domains td\n ON td.team_id = t.id\n AND td.deleted_at IS NULL\n AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))\nGROUP BY t.id, t.name, calendar_domain\nORDER BY t.name, calendar_domain;\n\nselect * from users u join calendars c on c.user_id = u.id\nwhere u.team_id = 882;\n\n\nselect * from activities where id = 74049485; # team 563 crm 537\nselect * from activities where id = 73272382; # team 563 crm 537\nselect * from activities where id = 64400389; # team 563 crm 537\nselect * from activities where id = 58081273; # team 563 crm 537\nselect * from activities where id = 54520297; # team 563 crm 537\nselect * from participants where activity_id = 58081273;\n\nselect * from activities where crm_configuration_id = 537 and provider = 'aircall'\nand account_id = 19003658 order by updated_at desc;\n\nselect * from contacts where crm_configuration_id = 537 and id = 35957759;\nselect * from accounts where crm_configuration_id = 537 and id = 19003658;\n\nselect * from automated_report_results where id = 1976;\nselect * from automated_reports where id = 583;\nselect * from activity_searches where id = 87714;\nselect * from activity_search_filters where activity_search_id = 87714;\n\nSELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid\nor uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot';\nselect * from rate_limits;\n\nselect * from automated_report_results where media_type = 'pdf' and status = 2\nand id IN (18, 1872);\nselect * from automated_reports where id = 54;\nSELECT * FROM users WHERE id IN (24623,29443,29613);\n\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\n\nselect * from text_relays where created_at > '2026-01-01';\nand id IN (32415, 32416);\n# and id = 32412;\n\nselect * from users where team_id = 2 and email like '%scott%' and id = 29510;\n\nSELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436\n\nSELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses\nFROM text_relays\nWHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')\nGROUP BY email_provider_id;\nSELECT id, status, telephony_provider_id, created_at\nFROM activities\nWHERE id IN (80028719, 80028846);\nSELECT id, status, code, email_sent_at, created_at, updated_at\nFROM text_relays\nWHERE id IN (32415, 32416);\nSELECT id, status, code, sender, recipient, created_at\nFROM text_relays\nWHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'\nORDER BY created_at DESC\nLIMIT 10;\n\nSELECT id, uuid, status, code, sender, recipient, created_at, updated_at\nFROM text_relays\nWHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');\n\n# ***************\nSELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count\nFROM users u\nINNER JOIN activities a ON u.id = a.user_id\nWHERE a.type LIKE 'sms%'\nAND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)\nGROUP BY u.id, u.email, u.name, u.softphone_number\nORDER BY sms_count DESC;\n\nselect * from teams where id = 1;\n\nselect * from roles;","depth":4,"on_screen":true,"value":"SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993\nSELECT * FROM users WHERE id = 25061;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 994;\nSELECT * FROM crm_profiles WHERE user_id = 25061;\n\nselect * from crm_configurations where id = 834;\nSELECT * FROM teams WHERE id = 882;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;\n\nSELECT * FROM contacts where crm_configuration_id = 834;\nSELECT * FROM opportunities WHERE team_id = 933\n# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');\nAND id IN (8482561,18352941,19042734,19232139,19445140,19472541);\nSELECT * FROM opportunity_contacts\nWHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 485; #\nSELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\nselect crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id\nwhere crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')\n# and l.converted_at IS NOT NULL\n;\n\n# ********************************************************************\nSELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')\nand opportunity_id IS NULL\norder by id desc;\n\nSELECT * FROM teams WHERE id = 604; # 598\nSELECT * FROM activities WHERE id = 74410828; # chelseaw@allvoices.co\nSELECT * FROM accounts WHERE id = 20068382;\nSELECT * FROM accounts WHERE id = 35186038;\n\nSELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 559 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;\nselect * from sidekick_settings where team_id = 781;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100\n\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 711;\nSELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL\nand is_internal = 0 and status = 'completed'\norder by id desc;\n\nSELECT * FROM crm_layout_entities\nWHERE crm_layout_id IN (2352, 2353);\n;\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 556 and sa.provider = 'hubspot';\n\nSELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;\nSELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;\nselect * from contacts\nwhere crm_configuration_id = 530\nand crm_provider_id = 872252;\n\nselect * from activities where crm_configuration_id = 530\nand user_id = 14343 and type like '%softphone%'\nand created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';\n\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya\nSELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);\n\n\nSELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t\nJOIN crm_configurations c ON t.id = c.team_id\nWHERE t.status = 'active';\n\nSELECT * FROM teams where id = 1091;\nSELECT * FROM crm_configurations where team_id = 1091;\nSELECT * FROM activity_providers where team_id = 1091;\nSELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT * FROM teams WHERE name LIKE '%Leadventure%';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1091 and sa.provider = 'salesforce';\n\nSELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812\nSELECT * FROM teams where id = 862;\nSELECT * FROM crm_configurations where team_id = 862;\nSELECT * FROM activity_providers where team_id = 862;\nSELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')\nand provider NOT IN ('hubspot', 'aircall')\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by id desc;\n\n\nSELECT t.id, crm.id, crm.provider, ap.* FROM teams t\njoin crm_configurations crm on t.id = crm.team_id\njoin activity_providers ap on t.id = ap.team_id\nwhere t.status = 'active' and ap.is_enabled = 1\nand crm.provider = 'hubspot'\nand ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',\n 'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');\n\nSELECT * FROM teams where id = 1068;\nSELECT * FROM crm_configurations where team_id = 1068;\nSELECT * FROM activity_providers where team_id = 1068;\n\nSELECT * FROM activities a\nwhere crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')\nand a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'\n )\n# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'\norder by a.id desc;\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1068 and sa.provider = 'hubspot';\n\n# ********************************************************************\n# ********************************************************************\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 933 and sa.provider = 'hubspot';\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262\nSELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 882 and sa.provider = 'hubspot';\nselect * from crm_layouts where crm_configuration_id = 834;\nselect * from crm_layout_entities where crm_layout_id = 2780;\nselect * from crm_fields where id IN (321153,321192,321193,321194);\n\nSELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1057 and sa.provider = 'hubspot';\n\nSELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8\n\nSELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20\n\nSELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last\n\nSELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10\n\nSELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2\n\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;\n\nSELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th\n# ********************************************************************\nSELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #\n\nSELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;\nselect * from users where team_id = 51; # 7783\nSELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130\nselect * from activity_searches where user_id = 7783;\nselect * from activity_search_filters where activity_search_id IN (32291, 32292);\n\nSELECT asf.activity_search_id, asf.id, asf.value\nFROM activity_search_filters asf\nWHERE asf.filter = 'group_id'\nAND asf.value IN (\n SELECT CONCAT(\n HEX(SUBSTR(uuid, 5, 4)), '-',\n HEX(SUBSTR(uuid, 3, 2)), '-',\n HEX(SUBSTR(uuid, 1, 2)), '-',\n HEX(SUBSTR(uuid, 9, 2)), '-',\n HEX(SUBSTR(uuid, 11))\n )\n FROM groups\n WHERE deleted_at IS NOT NULL\n);\n\nSELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th\n# ********************************************************************\nSELECT * FROM crm_configurations where provider = 'hubspot';\nSELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133\nSELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;\nSELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null\n# ********************************************************************\n\nselect * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';\nselect\n cp.*\n# DISTINCT t.id\n# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields\nFROM crm_profiles cp\nJOIN crm_configurations crm on crm.id = cp.crm_configuration_id\nJOIN users u on u.id = cp.user_id\nJOIN teams t ON t.id = crm.team_id\nWHERE crm.provider = 'salesforce' and t.status = 'active'\n and cp.archived_at IS NULL and u.deleted_at IS NULL\n and t.id NOT IN (1093)\n and t.id = 2\n and cp.contact_fields IS NULL;\n# and c.crm_provider_id = '003Uu00000ojD4NIAU';\n\nSELECT * FROM users WHERE id = 26484;\nSELECT * FROM crm_profiles WHERE user_id = 26484;\nSELECT * FROM social_accounts WHERE sociable_id = 26484;\nSELECT * FROM crm_configurations where provider = 'salesforce';\nselect * from users where id IN (10022, 10403);\nselect * from users where team_id IN (526);\nselect * from teams where id IN (526, 532);\nselect * from crm_configurations where id IN (500, 516);\nselect * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);\nselect * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 526 and sa.provider = 'salesforce';\nselect * from team_settings where team_id IN (526, 532);\n\nselect * from users where id IN (22824);\nselect * from crm_profiles where crm_configuration_id IN (1026);\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1093 and sa.provider = 'salesforce';\n\nselect * from teams where id = 1099;\nselect * from users where id = 29643\n\nselect * from activity_processing_states;\n\nSELECT * FROM teams where name LIKE '%Fare%'; # 233\nSELECT * FROM opportunities where crm_configuration_id = 215\n# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'\n;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1088 and sa.provider = 'hubspot';\n\nSELECT * FROM teams order by updated_at DESC\nSELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account\n\nselect * from crm_configurations where provider = 'pipedrive';\n\nselect * from teams where id = 957;\nselect * from crm_configurations where id = 957;\n\nSELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743\nSELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;\n\nselect * from users where team_id = 1; # 26726 - Gabriela Dureva\nSELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific\nselect * from activities where user_id = 26726 order by id desc;\nselect * from contacts where crm_configuration_id = 1\nand email IN ('charlotte.ward@prolific.com', 'frankie.bryant@prolific.com'); # 2094416, 2093620\nSELECT * FROM contacts WHERE id = 6284931;\n\nSELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id\nWHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;\n\nselect * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);\nselect * from crm_configurations where id = 1;\n\n43801692-1aeb-32ce-acba-5b80a479701a\n44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b\n405975c0-b3d0-7aaa-821f-09d59cae6dd1\n4caf848d-4bed-2299-b248-7788d41f9fca\n49bedc3f-f196-eef3-89c3-dea6a3b4aa63\n43420989-a09d-b8f8-9806-c8bbf7a02aac\n\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nSELECT * FROM activities WHERE id = 75461988;\n\nSELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;\n\nselect * from contacts where id = 17900517;\n\nselect * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id\nwhere crm.provider != 'salesforce';\n\nselect * from users where id = 21047;\nSELECT * FROM crm_configurations WHERE id = 892;\nSELECT * FROM teams WHERE id = 942;\nselect * from opportunities where team_id = 942 order by updated_at desc;\nselect * from contacts where team_id = 942 order by updated_at desc;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 942 and sa.provider = 'hubspot';\n\nSELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430\nSELECT * FROM crm_configurations WHERE id = 1;\nSELECT * FROM teams WHERE crm_id = 1;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1 and sa.provider = 'salesforce';\n\nselect id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1\nSELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430\n\nselect * from teams where id = 852;\nselect * from groups where id = 2286;\nselect * from sidekick_settings where team_id = 852;\nselect * from default_activity_types where team_id = 852;\n\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1 AND u.deleted_at IS NULL\nAND u.crm_required = 1\nAND u.team_id = 1\nORDER BY u.team_id;\n\nSELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (\n18481\n );\n\nSELECT cc.provider, cc.id, p.id, u.*\nFROM users u\nLEFT JOIN crm_profiles p ON u.id = p.user_id\nINNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'\nINNER JOIN crm_configurations cc ON t.crm_id = cc.id\nWHERE u.status = 1\n AND u.deleted_at IS NULL\n AND u.crm_required = 1\n# AND u.team_id = 1\n AND p.id IS NULL -- Move this condition to WHERE clause\nORDER BY u.team_id;\n\nSELECT * FROM opportunities WHERE id = 20002609;\nselect * from teams where id = 1122; # Velatir, 29953 - christian@velatir.com\nselect * from crm_configurations where id = 1060;\nselect * from crm_layouts where crm_configuration_id = 1060;\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 1122 and sa.provider = 'hubspot';\nselect * from opportunities where team_id = 1122 order by updated_at desc;\n\nselect * from crm_field_data where object_type = 'contact';\n\nSELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 248 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS\nSELECT * FROM users where id = 24115;\nSELECT * FROM accounts where id = 4002896;\nSELECT * FROM teams WHERE name LIKE '%adswerve%';\nSELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN (\"0069N000003GIQ9QAO\",\"0061r000019yGP9AAM\",\"0066900001S2KWlAAN\",\"0066900001TDpj2AAD\",\"0066900001b8uEwAAI\",\"0069N000001rQi0QAE\",\"006QF00000KD40mYAD\",\"006QF00000LzpRJYAZ\",\"0069N000002uomtQAA\",\"0069N000002xlMLQAY\",\"0066900001NV6ubAAD\",\"0061r00001HJp45AAD\",\"006QF00000uTlUoYAK\",\"006QF00000v0bZqYAI\");\nSELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203\n\nSELECT u.id, u.email, ac.name, a.* FROM activities a\nJOIN users u ON a.user_id = u.id\nJOIN accounts ac ON a.account_id = ac.id\nWHERE\nuuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or\nuuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or\nuuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;\n\nselect * from users where id = 5825;\nSELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;\n\nselect * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;\n19594, 862\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 862 and sa.provider = 'salesforce';\n\nselect * from automated_reports where id = 36;\nselect ar.frequency, r.*, ar.* from automated_report_results r\njoin automated_reports ar on r.report_id = ar.id\nwhere ar.frequency != 'one_off';\n\nselect s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;\nselect * from nudges n where n.activity_search_id\n\nselect * from teams where created_at > '2026-03-09';\nSELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065\nSELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;\n\nselect * from users where team_id = 1 and name like '%Lukas%'; # 7160\n\nSELECT * FROM teams WHERE id = 575;\nselect * from opportunities where team_id = 575;\nSELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,\nselect * from opportunities where team_id = 1126;\nSELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,\nselect * from opportunities where team_id = 1125;\nselect * from contacts c\nwhere c.team_id = 882;\n\nSELECT * FROM activities WHERE id = 76822967;\nSELECT * FROM crm_profiles WHERE user_id = 15440;\nSELECT * FROM crm_profiles WHERE crm_configuration_id = 555;\nSELECT * FROM crm_configurations WHERE id = 555;\nSELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 581 and sa.provider = 'salesforce';\n\nSELECT * FROM automated_report_results order by id desc;\n\nselect * from features;\nselect * from team_features where feature_id = 40;\n\nselect * from teams where id = 556;\n\nselect * from automated_reports;\nwhere id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , [\"pdf\",\"podcast\"]\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\nselect * from automated_report_results order by id desc;\nSELECT * FROM automated_report_results WHERE id = 1919;\n\nselect * from automated_report_results WHERE report_id = 54;\n\nselect * from opportunities where id = 7594349;\n\nSELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - jiminnyintegration@lesmills.com\nselect * from playbooks where team_id = 711; # event 226147\nSELECT * FROM playbook_categories WHERE playbook_id = 5515;\nSELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';\nSELECT * FROM crm_fields WHERE id = 226147;\nSELECT * FROM crm_field_values WHERE crm_field_id = 226147;\n\nSELECT * FROM crm_configurations WHERE id = 692;\nSELECT\n CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,\n u.email,\n sa.*,\n t.owner_id FROM social_accounts sa\nJOIN users u on u.id = sa.sociable_id\nJOIN teams t on t.id = u.team_id\nWHERE u.team_id = 711 and sa.provider = 'salesforce';\n\nSELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;\n\nselect * from leads;\n\nselect * from calendars;\n\nSELECT\n t.id AS team_id,\n t.name,\n LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain\nFROM teams t\nJOIN users u ON u.team_id = t.id\nJOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'\nLEFT JOIN team_domains td\n ON td.team_id = t.id\n AND td.deleted_at IS NULL\n AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))\nGROUP BY t.id, t.name, calendar_domain\nORDER BY t.name, calendar_domain;\n\nselect * from users u join calendars c on c.user_id = u.id\nwhere u.team_id = 882;\n\n\nselect * from activities where id = 74049485; # team 563 crm 537\nselect * from activities where id = 73272382; # team 563 crm 537\nselect * from activities where id = 64400389; # team 563 crm 537\nselect * from activities where id = 58081273; # team 563 crm 537\nselect * from activities where id = 54520297; # team 563 crm 537\nselect * from participants where activity_id = 58081273;\n\nselect * from activities where crm_configuration_id = 537 and provider = 'aircall'\nand account_id = 19003658 order by updated_at desc;\n\nselect * from contacts where crm_configuration_id = 537 and id = 35957759;\nselect * from accounts where crm_configuration_id = 537 and id = 19003658;\n\nselect * from automated_report_results where id = 1976;\nselect * from automated_reports where id = 583;\nselect * from activity_searches where id = 87714;\nselect * from activity_search_filters where activity_search_id = 87714;\n\nSELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid\nor uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;\n\nSELECT * FROM crm_configurations WHERE provider = 'hubspot';\nselect * from rate_limits;\n\nselect * from automated_report_results where media_type = 'pdf' and status = 2\nand id IN (18, 1872);\nselect * from automated_reports where id = 54;\nSELECT * FROM users WHERE id IN (24623,29443,29613);\n\nSELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;\n\nselect * from text_relays where created_at > '2026-01-01';\nand id IN (32415, 32416);\n# and id = 32412;\n\nselect * from users where team_id = 2 and email like '%scott%' and id = 29510;\n\nSELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436\n\nSELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses\nFROM text_relays\nWHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')\nGROUP BY email_provider_id;\nSELECT id, status, telephony_provider_id, created_at\nFROM activities\nWHERE id IN (80028719, 80028846);\nSELECT id, status, code, email_sent_at, created_at, updated_at\nFROM text_relays\nWHERE id IN (32415, 32416);\nSELECT id, status, code, sender, recipient, created_at\nFROM text_relays\nWHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'\nORDER BY created_at DESC\nLIMIT 10;\n\nSELECT id, uuid, status, code, sender, recipient, created_at, updated_at\nFROM text_relays\nWHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');\n\n# ***************\nSELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count\nFROM users u\nINNER JOIN activities a ON u.id = a.user_id\nWHERE a.type LIKE 'sms%'\nAND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)\nGROUP BY u.id, u.email, u.name, u.softphone_number\nORDER BY sms_count DESC;\n\nselect * from teams where id = 1;\n\nselect * from roles;","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"Project","depth":3,"on_screen":false,"role_description":"text"},{"role":"AXButton","text":"Project","depth":3,"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"New File or Directory…","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Expand Selected","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Collapse All","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Options","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.0,"top":0.0,"width":0.018055556,"height":0.026666667},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false}]...
|
6081160626368098919
|
2218652917440067143
|
visual_change
|
accessibility
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
41
1
40
65
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM teams WHERE name LIKE '%litify%'; # 1069, 994, 24993
SELECT * FROM users WHERE id = 25061;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 994;
SELECT * FROM crm_profiles WHERE user_id = 25061;
select * from crm_configurations where id = 834;
SELECT * FROM teams WHERE id = 882;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations WHERE provider = 'hubspot' and crm_provider_id = 7270388;
SELECT * FROM contacts where crm_configuration_id = 834;
SELECT * FROM opportunities WHERE team_id = 933
# AND crm_provider_id IN ('20131586060','46017317898','52543911090','53451356564','54101251892','54323768459');
AND id IN (8482561,18352941,19042734,19232139,19445140,19472541);
SELECT * FROM opportunity_contacts
WHERE opportunity_id IN (8482561,18352941,19042734,19232139,19445140,19472541);
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 485; #
SELECT * FROM opportunities WHERE team_id = 933 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
select crm.provider, l.* from leads l join crm_configurations crm on l.crm_configuration_id = crm.id
where crm.provider NOT IN ('salesforce', 'integration-app', 'bullhorn', 'copper')
# and l.converted_at IS NOT NULL
;
# [PASSWORD_DOTS]
SELECT * FROM activities a WHERE type IN ('email-inbound', 'email-outbound')
and opportunity_id IS NULL
order by id desc;
SELECT * FROM teams WHERE id = 604; # 598
SELECT * FROM activities WHERE id = 74410828; # [EMAIL]
SELECT * FROM accounts WHERE id = 20068382;
SELECT * FROM accounts WHERE id = 35186038;
SELECT * FROM contacts WHERE team_id = 852 and updated_at > '2026-01-23 12:30:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 559 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('cb6342b6-a183-401c-b0af-ede92b2ae763') = uuid;
select * from sidekick_settings where team_id = 781;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 26651871; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 7562435;
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8420347; # opflit 2100
SELECT * FROM crm_layouts WHERE crm_configuration_id = 711;
SELECT * FROM activities where crm_configuration_id = 711 and crm_provider_id IS NULL
and is_internal = 0 and status = 'completed'
order by id desc;
SELECT * FROM crm_layout_entities
WHERE crm_layout_id IN (2352, 2353);
;
SELECT * FROM crm_configurations where provider = 'hubspot' and id = 530;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 556 and sa.provider = 'hubspot';
SELECT * FROM activities WHERE uuid_to_bin('c6ca4b22-7738-4563-a95d-b8a9598924ae') = uuid;
SELECT * FROM activities WHERE uuid_to_bin('442abb2b-28bd-4be8-9c25-19e9bf02766d') = uuid;
select * from contacts
where crm_configuration_id = 530
and crm_provider_id = 872252;
select * from activities where crm_configuration_id = 530
and user_id = 14343 and type like '%softphone%'
and created_at between '2026-01-28 15:00:00' and '2026-01-28 15:10:00';
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 25666868; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id = 8646335; # Teya
SELECT * FROM crm_configurations where provider = 'hubspot' and crm_provider_id IN (5933397);
SELECT t.name, t.id, t.owner_id, c.id, c.provider, c.crm_base_url FROM teams t
JOIN crm_configurations c ON t.id = c.team_id
WHERE t.status = 'active';
SELECT * FROM teams where id = 1091;
SELECT * FROM crm_configurations where team_id = 1091;
SELECT * FROM activity_providers where team_id = 1091;
SELECT * FROM activities where crm_configuration_id = 1024 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT * FROM teams WHERE name LIKE '%Leadventure%';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1091 and sa.provider = 'salesforce';
SELECT * FROM teams WHERE name LIKE '%Wilson%'; # 862, 812
SELECT * FROM teams where id = 862;
SELECT * FROM crm_configurations where team_id = 862;
SELECT * FROM activity_providers where team_id = 862;
SELECT * FROM activities where crm_configuration_id = 812 and type IN ('softphone', 'softphone-outbound')
and provider NOT IN ('hubspot', 'aircall')
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by id desc;
SELECT t.id, crm.id, crm.provider, ap.* FROM teams t
join crm_configurations crm on t.id = crm.team_id
join activity_providers ap on t.id = ap.team_id
where t.status = 'active' and ap.is_enabled = 1
and crm.provider = 'hubspot'
and ap.provider NOT IN ('hubspot', 'aircall', 'uploader', 'gong', 'twilio', 'zoom-bot', 'google-meet', 'ms-teams',
'outreach', 'close', 'ringcentral', 'dialpad', 'zoom-phone');
SELECT * FROM teams where id = 1068;
SELECT * FROM crm_configurations where team_id = 1068;
SELECT * FROM activity_providers where team_id = 1068;
SELECT * FROM activities a
where crm_configuration_id = 993 and type IN ('softphone', 'softphone-outbound')
and a.provider NOT IN ('hubspot', 'uploader', 'gong', 'twilio', 'google-meet', 'ms-teams','close'
)
# and telephony_provider_id = '019c1131-a22f-4792-b9ea-20adf6a02ed0'
order by a.id desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1068 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 882; # 933 - GoGlobal , portalId: 6017093
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 933 and updated_at > '2026-02-06 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 933 and sa.provider = 'hubspot';
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 834; # 882 - AnyVan , portalId: 5468262
SELECT * FROM contacts WHERE crm_configuration_id = 834 and updated_at > '2026-03-30 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and updated_at > '2026-03-04 08:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 882 and sa.provider = 'hubspot';
select * from crm_layouts where crm_configuration_id = 834;
select * from crm_layout_entities where crm_layout_id = 2780;
select * from crm_fields where id IN (321153,321192,321193,321194);
SELECT * FROM opportunities WHERE crm_configuration_id = 834 and id = 10993426;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 988; # 1057 - Teya (543ce4f4-168c-4571-91ea-5b35c253f06f) , portalId: 26651871
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1057 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1057 and sa.provider = 'hubspot';
SELECT * FROM crm_configurations where id = 533; # 559 - Connectd , portalId: 6710988
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 559 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 801; # 852 - Rise Vision , portalId: 2700250
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 852 and updated_at > '2026-02-04 00:00:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 962; # 1034 - evergrowth.io , portalId: 143180990
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1034 and updated_at > '2026-02-04 00:00:00' order by updated_at desc;
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 1037; # 1102 - Jibble , portalId: 6649755
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1102 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 8
SELECT * FROM crm_configurations where id = 1015; # 1049 - Travefy , portalId: 48904401
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1049 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 20
SELECT * FROM crm_configurations where id = 64; # 70 - SalaryFinance , portalId: 3404115
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 70 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 6th last
SELECT * FROM crm_configurations where id = 802; # 853 - Street Group , portalId: 7658438
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 853 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 10
SELECT * FROM crm_configurations where id = 872; # 921 - In Professional Development , portalId: 9238273
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 921 and updated_at > '2026-02-04 12:30:00' order by updated_at desc; # 2
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 550; # 576 - SeedLegals , portalId: 3028661
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 576 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 989; # 1058 - rtaoutdoor.com , portalId: 22371204
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1058 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 896; # 946 - Mintago , portalId: 6621281
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 946 and updated_at > '2026-02-05 14:00:00' order by updated_at desc;
SELECT * FROM crm_configurations where id = 617; # 641 - PCS , portalId: 5244937
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 641 and updated_at > '2026-02-05 14:00:00' order by updated_at desc; # 7th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where id = 649; # 670 - Eventeny , portalId: 4492849
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-18 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 670 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; #
SELECT * FROM crm_configurations where id = 48; # 51 - CleanCloud , portalId: 4373137
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-03-04 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 51 and updated_at > '2026-02-09 08:00:00' order by updated_at desc;
select * from users where team_id = 51; # 7783
SELECT * FROM groups WHERE uuid_to_bin('8a8d2cb6-8b55-4fa3-8b5c-5f0e3d8de59a') = uuid; # 1130
select * from activity_searches where user_id = 7783;
select * from activity_search_filters where activity_search_id IN (32291, 32292);
SELECT asf.activity_search_id, asf.id, asf.value
FROM activity_search_filters asf
WHERE asf.filter = 'group_id'
AND asf.value IN (
SELECT CONCAT(
HEX(SUBSTR(uuid, 5, 4)), '-',
HEX(SUBSTR(uuid, 3, 2)), '-',
HEX(SUBSTR(uuid, 1, 2)), '-',
HEX(SUBSTR(uuid, 9, 2)), '-',
HEX(SUBSTR(uuid, 11))
)
FROM groups
WHERE deleted_at IS NOT NULL
);
SELECT * FROM crm_configurations where id = 272; # 290 - Bonham & Brook , portalId: 5705856
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-05 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 290 and updated_at > '2026-02-09 08:00:00' order by updated_at desc; # 6th
# [PASSWORD_DOTS]
SELECT * FROM crm_configurations where provider = 'hubspot';
SELECT * FROM crm_configurations where id = 1056; # 1119 - Chromatic , portalId: 45602133
SELECT * FROM opportunities WHERE team_id = 1119 and remotely_created_at > '2026-02-01 00:00:00' order by updated_at desc;
SELECT * FROM opportunities WHERE team_id = 1119 and updated_at > '2026-02-09 09:00:00' order by updated_at desc; # null
# [PASSWORD_DOTS]
select * from contacts where crm_provider_id = '003Uu00000ojD4NIAU';
select
cp.*
# DISTINCT t.id
# cp.id, cp.user_id, t.id, cp.crm_configuration_id, cp.contact_fields
FROM crm_profiles cp
JOIN crm_configurations crm on crm.id = cp.crm_configuration_id
JOIN users u on u.id = cp.user_id
JOIN teams t ON t.id = crm.team_id
WHERE crm.provider = 'salesforce' and t.status = 'active'
and cp.archived_at IS NULL and u.deleted_at IS NULL
and t.id NOT IN (1093)
and t.id = 2
and cp.contact_fields IS NULL;
# and c.crm_provider_id = '003Uu00000ojD4NIAU';
SELECT * FROM users WHERE id = 26484;
SELECT * FROM crm_profiles WHERE user_id = 26484;
SELECT * FROM social_accounts WHERE sociable_id = 26484;
SELECT * FROM crm_configurations where provider = 'salesforce';
select * from users where id IN (10022, 10403);
select * from users where team_id IN (526);
select * from teams where id IN (526, 532);
select * from crm_configurations where id IN (500, 516);
select * from crm_profiles where crm_configuration_id IN (500, 516) and user_id IN (10022, 10403);
select * from contacts where crm_configuration_id IN (500, 516) and crm_provider_id = '003Uu00000ojD4NIAU';
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 526 and sa.provider = 'salesforce';
select * from team_settings where team_id IN (526, 532);
select * from users where id IN (22824);
select * from crm_profiles where crm_configuration_id IN (1026);
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1093 and sa.provider = 'salesforce';
select * from teams where id = 1099;
select * from users where id = 29643
select * from activity_processing_states;
SELECT * FROM teams where name LIKE '%Fare%'; # 233
SELECT * FROM opportunities where crm_configuration_id = 215
# and crm_provider_id = 'oppo_ogESZf2P50nDrd1nGPvKDXeA6sSaTN5v51Lp4ayVzKR'
;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1088 and sa.provider = 'hubspot';
SELECT * FROM teams order by updated_at DESC
SELECT * FROM crm_configurations WHERE id = 1019; # SimpleConsign 1088 - no social account
select * from crm_configurations where provider = 'pipedrive';
select * from teams where id = 957;
select * from crm_configurations where id = 957;
SELECT * FROM teams WHERE name LIKE '%Prolific%'; # 544, 518, 10743
SELECT * FROM opportunities where crm_configuration_id = 518 order by id desc;
select * from users where team_id = 1; # 26726 - Gabriela Dureva
SELECT * FROM opportunities where user_id = 26726; # 16834447 - Prolific
select * from activities where user_id = 26726 order by id desc;
select * from contacts where crm_configuration_id = 1
and email IN ('[EMAIL]', '[EMAIL]'); # 2094416, 2093620
SELECT * FROM contacts WHERE id = 6284931;
SELECT p.* FROM activities a JOIN participants p ON a.id = p.activity_id
WHERE a.user_id = 26726 and p.lead_id IN (2094416, 2093620) and a.created_at > '2026-01-01 00:00:00' order by p.email;
select * from activities where id IN (75509259,75509261,75509261,75511034,75026464,75517602,75517605);
select * from crm_configurations where id = 1;
43801692-1aeb-32ce-acba-5b80a479701a
44c3c9cf-6f5e-75f3-8179-bc9f75dd2b1b
405975c0-b3d0-7aaa-821f-09d59cae6dd1
4caf848d-4bed-2299-b248-7788d41f9fca
49bedc3f-f196-eef3-89c3-dea6a3b4aa63
43420989-a09d-b8f8-9806-c8bbf7a02aac
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
SELECT * FROM activities WHERE id = 75461988;
SELECT * FROM activities WHERE uuid_to_bin('d6c5052e-e972-49e9-8912-26f2f7d6c5f6') = uuid;
select * from contacts where id = 17900517;
select * from contact_roles cr join crm_configurations crm on cr.crm_configuration_id = crm.id
where crm.provider != 'salesforce';
select * from users where id = 21047;
SELECT * FROM crm_configurations WHERE id = 892;
SELECT * FROM teams WHERE id = 942;
select * from opportunities where team_id = 942 order by updated_at desc;
select * from contacts where team_id = 942 order by updated_at desc;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 942 and sa.provider = 'hubspot';
SELECT * FROM opportunities where team_id = 1 and crm_provider_id IN ('006Pq00000NeH6XIAV', '006Pq000007z8kdIAA'); # 10697889, 6621430
SELECT * FROM crm_configurations WHERE id = 1;
SELECT * FROM teams WHERE crm_id = 1;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1 and sa.provider = 'salesforce';
select id, user_id, opportunity_fields from crm_profiles where crm_configuration_id = 1
SELECT * FROM opportunities where team_id = 1 order by updated_at desc; # 10697889, 6621430
select * from teams where id = 852;
select * from groups where id = 2286;
select * from sidekick_settings where team_id = 852;
select * from default_activity_types where team_id = 852;
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id AND p.id IS NULL -- no profile
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active' -- team is active
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1 AND u.deleted_at IS NULL
AND u.crm_required = 1
AND u.team_id = 1
ORDER BY u.team_id;
SELECT * FROM crm_profiles cp where cp.crm_configuration_id = 1 and cp.user_id IN (
18481
);
SELECT cc.provider, cc.id, p.id, u.*
FROM users u
LEFT JOIN crm_profiles p ON u.id = p.user_id
INNER JOIN teams t ON u.team_id = t.id AND t.status = 'active'
INNER JOIN crm_configurations cc ON t.crm_id = cc.id
WHERE u.status = 1
AND u.deleted_at IS NULL
AND u.crm_required = 1
# AND u.team_id = 1
AND p.id IS NULL -- Move this condition to WHERE clause
ORDER BY u.team_id;
SELECT * FROM opportunities WHERE id = 20002609;
select * from teams where id = 1122; # Velatir, 29953 - [EMAIL]
select * from crm_configurations where id = 1060;
select * from crm_layouts where crm_configuration_id = 1060;
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3596;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 1122 and sa.provider = 'hubspot';
select * from opportunities where team_id = 1122 order by updated_at desc;
select * from crm_field_data where object_type = 'contact';
SELECT * FROM activities WHERE uuid_to_bin('374fc8ed-3315-4c9f-9b25-318b7fd2928f') = uuid; # 76584262
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 248 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles where user_id = 24115; # 005QF000002CswMYAS
SELECT * FROM users where id = 24115;
SELECT * FROM accounts where id = 4002896;
SELECT * FROM teams WHERE name LIKE '%adswerve%';
SELECT * FROM opportunities where crm_configuration_id = 230 AND crm_provider_id IN ("0069N000003GIQ9QAO","0061r000019yGP9AAM","0066900001S2KWlAAN","0066900001TDpj2AAD","0066900001b8uEwAAI","0069N000001rQi0QAE","006QF00000KD40mYAD","006QF00000LzpRJYAZ","0069N000002uomtQAA","0069N000002xlMLQAY","0066900001NV6ubAAD","0061r00001HJp45AAD","006QF00000uTlUoYAK","006QF00000v0bZqYAI");
SELECT * FROM opportunities WHERE crm_configuration_id = 230 AND crm_provider_id = '0069N000003GIQ9QAO'; # 6272203
SELECT u.id, u.email, ac.name, a.* FROM activities a
JOIN users u ON a.user_id = u.id
JOIN accounts ac ON a.account_id = ac.id
WHERE
uuid_to_bin('e3269598-b562-44fb-b5e9-9d2694dc63e0') = a.uuid or
uuid_to_bin('66ddc3ab-4e15-45aa-af0c-248c1eece593') = a.uuid or
uuid_to_bin('826bd328-e1cc-4213-b8d8-572454cacc07') = a.uuid;
select * from users where id = 5825;
SELECT * FROM activities WHERE uuid_to_bin('e56aa2e8-231a-421b-ab1f-cb38ed2bf573') = uuid;
select * from activities where uuid_to_bin('91e13b2f-2d1b-45f8-b1fd-1141b6563782') = uuid;
19594, 862
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 862 and sa.provider = 'salesforce';
select * from automated_reports where id = 36;
select ar.frequency, r.*, ar.* from automated_report_results r
join automated_reports ar on r.report_id = ar.id
where ar.frequency != 'one_off';
select s.* from activity_searches s join users u ON s.user_id = u.id where u.team_id = 882;
select * from nudges n where n.activity_search_id
select * from teams where created_at > '2026-03-09';
SELECT * FROM crm_layouts WHERE crm_configuration_id = 1065; # 1065
SELECT * FROM crm_layout_entities WHERE crm_layout_id = 3617;
select * from users where team_id = 1 and name like '%Lukas%'; # 7160
SELECT * FROM teams WHERE id = 575;
select * from opportunities where team_id = 575;
SELECT * FROM teams WHERE name LIKE '%Integrum ESG%'; # 1126, 1065,
select * from opportunities where team_id = 1126;
SELECT * FROM teams WHERE name LIKE '%Base%'; # 1125, 1063,
select * from opportunities where team_id = 1125;
select * from contacts c
where c.team_id = 882;
SELECT * FROM activities WHERE id = 76822967;
SELECT * FROM crm_profiles WHERE user_id = 15440;
SELECT * FROM crm_profiles WHERE crm_configuration_id = 555;
SELECT * FROM crm_configurations WHERE id = 555;
SELECT * FROM users WHERE id = 15440; # team. 581, gr. 15440, pl. 3911, act. field 162182
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 581 and sa.provider = 'salesforce';
SELECT * FROM automated_report_results order by id desc;
select * from features;
select * from team_features where feature_id = 40;
select * from teams where id = 556;
select * from automated_reports;
where id = 54; # 4fdd41f6-dcf0-30d0-b339-7345381b6044 , ["pdf","podcast"]
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from automated_report_results order by id desc;
SELECT * FROM automated_report_results WHERE id = 1919;
select * from automated_report_results WHERE report_id = 54;
select * from opportunities where id = 7594349;
SELECT * FROM teams WHERE name LIKE '%Les%'; # 711, 692, 16067 - [EMAIL]
select * from playbooks where team_id = 711; # event 226147
SELECT * FROM playbook_categories WHERE playbook_id = 5515;
SELECT * FROM crm_fields WHERE crm_configuration_id = 692 and object_type = 'event';
SELECT * FROM crm_fields WHERE id = 226147;
SELECT * FROM crm_field_values WHERE crm_field_id = 226147;
SELECT * FROM crm_configurations WHERE id = 692;
SELECT
CONCAT(u.id, CASE WHEN u.id = t.owner_id THEN ' (owner)' ELSE '' END) AS user_id,
u.email,
sa.*,
t.owner_id FROM social_accounts sa
JOIN users u on u.id = sa.sociable_id
JOIN teams t on t.id = u.team_id
WHERE u.team_id = 711 and sa.provider = 'salesforce';
SELECT * FROM crm_profiles cp JOIN users u on u.id = cp.user_id WHERE u.team_id = 711;
select * from leads;
select * from calendars;
SELECT
t.id AS team_id,
t.name,
LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1)) AS calendar_domain
FROM teams t
JOIN users u ON u.team_id = t.id
JOIN calendars c ON c.user_id = u.id AND c.status = 'active' AND c.calendar_provider_id LIKE '%@%'
LEFT JOIN team_domains td
ON td.team_id = t.id
AND td.deleted_at IS NULL
AND td.domain = LOWER(SUBSTRING_INDEX(c.calendar_provider_id, '@', -1))
GROUP BY t.id, t.name, calendar_domain
ORDER BY t.name, calendar_domain;
select * from users u join calendars c on c.user_id = u.id
where u.team_id = 882;
select * from activities where id = 74049485; # team 563 crm 537
select * from activities where id = 73272382; # team 563 crm 537
select * from activities where id = 64400389; # team 563 crm 537
select * from activities where id = 58081273; # team 563 crm 537
select * from activities where id = 54520297; # team 563 crm 537
select * from participants where activity_id = 58081273;
select * from activities where crm_configuration_id = 537 and provider = 'aircall'
and account_id = 19003658 order by updated_at desc;
select * from contacts where crm_configuration_id = 537 and id = 35957759;
select * from accounts where crm_configuration_id = 537 and id = 19003658;
select * from automated_report_results where id = 1976;
select * from automated_reports where id = 583;
select * from activity_searches where id = 87714;
select * from activity_search_filters where activity_search_id = 87714;
SELECT * FROM activities WHERE uuid_to_bin('8827f672-202d-4162-9d04-73ff5f0566a9') = uuid
or uuid_to_bin('47842446-af51-4bcb-854f-cc6560290101') = uuid;
SELECT * FROM crm_configurations WHERE provider = 'hubspot';
select * from rate_limits;
select * from automated_report_results where media_type = 'pdf' and status = 2
and id IN (18, 1872);
select * from automated_reports where id = 54;
SELECT * FROM users WHERE id IN (24623,29443,29613);
SELECT * FROM automated_report_results WHERE uuid_to_bin('822fa41b-afd3-43a9-a248-86b0e36f3131') = uuid;
select * from text_relays where created_at > '2026-01-01';
and id IN (32415, 32416);
# and id = 32412;
select * from users where team_id = 2 and email like '%scott%' and id = 29510;
SELECT * FROM activities WHERE uuid_to_bin('67cebfc2-ed56-44a2-8c68-7a0286ed8618') = uuid; # 79763436
SELECT email_provider_id, COUNT(*) as count, GROUP_CONCAT(id) as ids, GROUP_CONCAT(status) as statuses
FROM text_relays
WHERE email_provider_id IN ('19e2027868a64b42', '19e2033ed8ea6b10')
GROUP BY email_provider_id;
SELECT id, status, telephony_provider_id, created_at
FROM activities
WHERE id IN (80028719, 80028846);
SELECT id, status, code, email_sent_at, created_at, updated_at
FROM text_relays
WHERE id IN (32415, 32416);
SELECT id, status, code, sender, recipient, created_at
FROM text_relays
WHERE sender LIKE '%mario.georgiev%' OR sender LIKE '%stoyan.tomov%'
ORDER BY created_at DESC
LIMIT 10;
SELECT id, uuid, status, code, sender, recipient, created_at, updated_at
FROM text_relays
WHERE uuid = uuid_to_bin('0626141c-27a6-4d8c-aff8-7c8020a2c656');
# [PASSWORD_DOTS]
SELECT DISTINCT u.id, u.email, u.name, u.softphone_number, COUNT(a.id) as sms_count
FROM users u
INNER JOIN activities a ON u.id = a.user_id
WHERE a.type LIKE 'sms%'
AND a.created_at > DATE_SUB(NOW(), INTERVAL 30 DAY)
GROUP BY u.id, u.email, u.name, u.softphone_number
ORDER BY sms_count DESC;
select * from teams where id = 1;
select * from roles;
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
NULL
|
NULL
|
NULL
|
NULL
|
|
64668
|
2265
|
50
|
2026-05-20T14:11:03.640194+00:00
|
/Users/lukas/.screenpipe/data/data/2026-05-20/1779 /Users/lukas/.screenpipe/data/data/2026-05-20/1779286263640_m2.jpg...
|
PhpStorm
|
faVsco.js – console [PROD]
|
1
|
NULL
|
monitor_2
|
NULL
|
NULL
|
NULL
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}...
|
[{"role":"AXButton","text" [{"role":"AXButton","text":"Project: faVsco.js, menu","depth":5,"bounds":{"left":0.025930852,"top":0.019952115,"width":0.03856383,"height":0.025538707},"on_screen":true,"help_text":"~/jiminny/app","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"#12105 on JY-20915-add-domain-specific-emai…relay, menu","depth":5,"bounds":{"left":0.064494684,"top":0.019952115,"width":0.124667555,"height":0.025538707},"on_screen":true,"help_text":"Pull request #12105 exists for current branch JY-20915-add-domain-specific-emai…relay","role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Start Listening for PHP Debug Connections","depth":5,"bounds":{"left":0.8450798,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"SmsReceivedTest","depth":6,"bounds":{"left":0.86037236,"top":0.019952115,"width":0.05518617,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Run 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9155585,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Debug 'SmsReceivedTest'","depth":6,"bounds":{"left":0.9268617,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"More Actions","depth":6,"bounds":{"left":0.9381649,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"JetBrains AI","depth":5,"bounds":{"left":0.96609044,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Search Everywhere","depth":5,"bounds":{"left":0.9773936,"top":0.019952115,"width":0.011303191,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"IDE and Project Settings","depth":5,"bounds":{"left":0.9886968,"top":0.019952115,"width":0.011303186,"height":0.025538707},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Sync Changes","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide This Notification","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":false,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Code changed:","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.042220745,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Hide","depth":4,"bounds":{"left":0.27027926,"top":1.0,"width":0.008643617,"height":0.0},"on_screen":false,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXStaticText","text":"3","depth":4,"bounds":{"left":0.38098404,"top":0.15003991,"width":0.007978723,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"3","depth":4,"bounds":{"left":0.39095744,"top":0.15003991,"width":0.007978723,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXStaticText","text":"12","depth":4,"bounds":{"left":0.40093085,"top":0.15003991,"width":0.009640957,"height":0.015163607},"on_screen":true,"role_description":"text"},{"role":"AXButton","text":"Previous Highlighted Error","depth":4,"bounds":{"left":0.41223404,"top":0.14844373,"width":0.00731383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXButton","text":"Next Highlighted Error","depth":4,"bounds":{"left":0.4195479,"top":0.14844373,"width":0.006981383,"height":0.018355945},"on_screen":true,"role_description":"button","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false},{"role":"AXTextArea","text":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","depth":4,"on_screen":true,"value":"<?php\n\ndeclare(strict_types=1);\n\nnamespace Tests\\Unit\\Notifications\\Activities;\n\nuse Illuminate\\Support\\Facades\\Config;\nuse Jiminny\\Mail\\Activities\\SmsReceived as SmsMailable;\nuse Jiminny\\Models\\Activity;\nuse Jiminny\\Notifications\\Activities\\SmsReceived;\nuse Mockery;\nuse PHPUnit\\Framework\\Attributes\\CoversClass;\nuse ReflectionClass;\nuse Tests\\TestCase;\nuse Vinkla\\Hashids\\Facades\\Hashids;\n\n#[CoversClass(SmsReceived::class)]\nclass SmsReceivedTest extends TestCase\n{\n private const int ACTIVITY_ID = 123;\n private const int USER_ID = 456;\n private const string FROM_PHONE = '+1234567890';\n private const string TO_PHONE = '+0987654321';\n private const string PROSPECT_NAME = 'John Doe';\n private const string NOTIFIABLE_EMAIL = 'user@example.com';\n private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';\n\n private $activity;\n private $outboundActivity;\n private $fromParticipant;\n private $toParticipant;\n\n protected function setUp(): void\n {\n parent::setUp();\n Mockery::close();\n\n $this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];\n $this->toParticipant = (object) ['phone_number' => self::TO_PHONE];\n\n $this->activity = Mockery::mock(Activity::class)->makePartial();\n $this->activity->id = self::ACTIVITY_ID;\n $this->activity->user_id = self::USER_ID;\n $this->activity->prospect_name = self::PROSPECT_NAME;\n $this->activity->from = $this->fromParticipant;\n $this->activity->to = $this->toParticipant;\n\n $this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();\n\n Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);\n Config::set('jiminny.deploy_region', 'us');\n }\n\n protected function tearDown(): void\n {\n Mockery::close();\n parent::tearDown();\n }\n\n public function testConstructorStoresActivity(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $property = $reflection->getProperty('activity');\n $property->setAccessible(true);\n\n $this->assertSame($this->activity, $property->getValue($notification));\n }\n\n public function testViaReturnsMail(): void\n {\n $notification = new SmsReceived($this->activity);\n $notifiable = $this->createMock(\\stdClass::class);\n\n $result = $notification->via($notifiable);\n\n $this->assertEquals(['mail'], $result);\n }\n\n public function testToMailReturnsMailable(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithUsRegion(): void\n {\n Config::set('jiminny.deploy_region', 'us');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testToMailWithEuRegion(): void\n {\n Config::set('jiminny.deploy_region', 'eu');\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n\n public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void\n {\n $notification = new SmsReceived($this->activity);\n\n $reflection = new ReflectionClass($notification);\n $method = $reflection->getMethod('getOutboundActivity');\n $method->setAccessible(true);\n\n $result = $method->invoke($notification);\n\n $this->assertNull($result);\n }\n\n public function testToMailCallsGetOutboundActivity(): void\n {\n Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');\n\n $notifiable = $this->createMock(\\stdClass::class);\n $notifiable->email = self::NOTIFIABLE_EMAIL;\n\n $notification = new SmsReceived($this->activity);\n\n $result = $notification->toMail($notifiable);\n\n $this->assertInstanceOf(SmsMailable::class, $result);\n }\n}","role_description":"text entry area","is_enabled":true,"is_focused":false,"is_selected":false,"is_expanded":false}]...
|
9048080624196372634
|
2190121752968985234
|
visual_change
|
accessibility
|
NULL
|
Project: faVsco.js, menu
#12105 on JY-20915-add-do Project: faVsco.js, menu
#12105 on JY-20915-add-domain-specific-emai…relay, menu
Start Listening for PHP Debug Connections
SmsReceivedTest
Run 'SmsReceivedTest'
Debug 'SmsReceivedTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
3
3
12
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Tests\Unit\Notifications\Activities;
use Illuminate\Support\Facades\Config;
use Jiminny\Mail\Activities\SmsReceived as SmsMailable;
use Jiminny\Models\Activity;
use Jiminny\Notifications\Activities\SmsReceived;
use Mockery;
use PHPUnit\Framework\Attributes\CoversClass;
use ReflectionClass;
use Tests\TestCase;
use Vinkla\Hashids\Facades\Hashids;
#[CoversClass(SmsReceived::class)]
class SmsReceivedTest extends TestCase
{
private const int ACTIVITY_ID = 123;
private const int USER_ID = 456;
private const string FROM_PHONE = '+1234567890';
private const string TO_PHONE = '+0987654321';
private const string PROSPECT_NAME = 'John Doe';
private const string NOTIFIABLE_EMAIL = '[EMAIL]';
private const string GOOGLE_TEXT_HOST = 'txt.jiminny.com';
private $activity;
private $outboundActivity;
private $fromParticipant;
private $toParticipant;
protected function setUp(): void
{
parent::setUp();
Mockery::close();
$this->fromParticipant = (object) ['phone_number' => self::FROM_PHONE];
$this->toParticipant = (object) ['phone_number' => self::TO_PHONE];
$this->activity = Mockery::mock(Activity::class)->makePartial();
$this->activity->id = self::ACTIVITY_ID;
$this->activity->user_id = self::USER_ID;
$this->activity->prospect_name = self::PROSPECT_NAME;
$this->activity->from = $this->fromParticipant;
$this->activity->to = $this->toParticipant;
$this->outboundActivity = Mockery::mock(Activity::class)->shouldIgnoreMissing();
Config::set('jiminny.google_text_host', self::GOOGLE_TEXT_HOST);
Config::set('jiminny.deploy_region', 'us');
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function testConstructorStoresActivity(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$property = $reflection->getProperty('activity');
$property->setAccessible(true);
$this->assertSame($this->activity, $property->getValue($notification));
}
public function testViaReturnsMail(): void
{
$notification = new SmsReceived($this->activity);
$notifiable = $this->createMock(\stdClass::class);
$result = $notification->via($notifiable);
$this->assertEquals(['mail'], $result);
}
public function testToMailReturnsMailable(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithUsRegion(): void
{
Config::set('jiminny.deploy_region', 'us');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testToMailWithEuRegion(): void
{
Config::set('jiminny.deploy_region', 'eu');
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
public function testGetOutboundActivityReturnsNullWhenNoOutboundFound(): void
{
$notification = new SmsReceived($this->activity);
$reflection = new ReflectionClass($notification);
$method = $reflection->getMethod('getOutboundActivity');
$method->setAccessible(true);
$result = $method->invoke($notification);
$this->assertNull($result);
}
public function testToMailCallsGetOutboundActivity(): void
{
Hashids::shouldReceive('encode')->with(self::ACTIVITY_ID)->andReturn('abc123');
$notifiable = $this->createMock(\stdClass::class);
$notifiable->email = self::NOTIFIABLE_EMAIL;
$notification = new SmsReceived($this->activity);
$result = $notification->toMail($notifiable);
$this->assertInstanceOf(SmsMailable::class, $result);
}
}...
|
NULL
|
NULL
|
NULL
|
NULL
|