|
15664
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15664
|
|
15663
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15663
|
|
15662
|
Project: faVsco.js, menu
PhostormVIewINavicareCode Project: faVsco.js, menu
PhostormVIewINavicareCodeKeractorFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitroledey© SyncRelatedActivityManager.php© BatchSyncCollectol© HubspotSyncStrategyBase.phpCachedcrmservicebecorator.onp© ProspectCache.phpe balchsynckealsseo closedDealstagess© MatchactivityermData.png© CrmActivityService.phgDealrielasservice.g© CrmObiectsResolver.phpc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements HubspotcllentintertaceC) FieldT vpeconvertee Hubspotclientinter* doaram bool sarchivedc) Hubspotlokenman© PayloadBuilder.phpC) RemotecrmobiectrP ResponseNormalize* @neturn array<0wner>|[]13 usagesc) Service,onoC)SvncFieldAction.onpublic function get0wnersArchived(bool $archived = true): arrayC) SvncRelatedActivitSendpoint ="crm/vs/owners':C) WebhookSvncBatclSoueryParams =v MintearationAorarchived" = Sarchived ? "true': "false',M AcceccorsSauervstrind = htto bulld auery squerv?arams)> D ConfigD DTO> M SiltersSowners = 1D Jobsh ProcnectSoarchStrtrySresnonse = Sthis->makeRequest(endnoint• Sendooint. auervStrina: SauervStrino)•W service lralts© DataClient.phpSresnonselata = Sresnonse?->toArravO.Console,LOg XChanaes 11 files= env.local aonTJ0 + → Side-by-side viewer •8 35f036ac app/Services/Crm/Hubspot/Client.phgDo not ignore© HandleHubspotRateLimit.php app/Jobs/Middleware© HubspotClientinterface.php app/Services/Crm/Hubspot© HubspotPaginationService.php app/Services/Crm/Hubspot/Pagination@ JiminnyDebugCommand.php app/Console/Commandsphp logging.php config© MatchActivityCrmData.php app/Jobs/Crm© MatchCrmData.php app/Jobs/Activity/Import© PaginationState.php app/Services/Crm/Hubspot/Pagination© RateLimitException.php app/ExceptionsUnversioned Files 9 files=.env.nikilocal appE.env.other app@) CanAccessAiReportsTest.php tests/Unit/Policies© CreateMockAskJiminnvReportResultCommand.php app/Console/Commands/Rep_6 favicon.ico publicEids.txt aprT raw sal querv.sal app@ SimulateWebhooksCommand.ohn app/Console/Commands/Crm/HubspotMJ WEBHOOK FILTERING IMPLEMENTATION.md apduse HubSpot\Client\ Crm\Pipelines Model\PipelineStaqeuse Hubspot Cllent Crm Properties Model Property:use Hubspot biscovery Discovery:Jiminny Exceptions CrmException:Jiminnv\Exceptions|SocialAccountTokenInvalidException:use Jaminny woos crm Notelbnect:use Jaminny Models Crm Field:use Jiminny Services Crm BaseCuient.~Jiminnv|Services|Crm|Hubspot > Clientpublic function __constructSocialAccountService $socialAccountService,HubsnotPaginationService SnaginationServico.HuhsnotTokenManaden StokenManagennarent:. construct/SsocialAccountService)•Sthis->naginationSenvice = SnaginationSenvico•Sthis->tokenManager = $tokenManager:olner Code will hoin INF to underctand vour Laravel ann code II Generate II Don't Show Anvmore (50 minutes aao)hell• suppont Dally • In 4h 03m100% C47 • Mon 11 May 10:07:30AskJiminnyReportActivityServiceTestv+0 ..— | A2 A65 X1X1A=custom.log~=laravel.logA SF (jiminny@localhost]4 HS_local (jiminny@localhost]# console [PKOb.# console leu)# console [slAGiNG)[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn"Transter-Encod1nq":"chunked")."Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncludeSubDomains: preload")naccent-encodino""access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\","x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"So+-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtm0V-1778163675-1.0.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Report-To":["{"endpoints\":[{"unl".httns:a.nel.cloudflane.com./nenon+|W/v42c=NVA1cVTPQfVm32anS0axVF/sd2R\"group)":\"cf-nel\","max_age\":604800}"],"NEL":["{success_traccion.0.01rreportto. "cr-nel,CascadeHubspot Rate Limit Re3. We just made an additional API call while already rate-limited - amplifying the pressure4. If matchByPhone returns nothing too, the activity gets persisted as "no CRM match" even though the contact exists in HubSpot. Wrong data, no signal to retryThe middleware on MatchActivityCrmData cannot help, because the 429 was already swallowed before any RateLimitException could be raised.how serious is it reallv?In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the AP| add-on), which is much more generous than the search endpoint (5 RPS). 429s on this endpoint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail• When it does hannen. the cost is silent data corruntion wrona CRM match) olus extra wasted APl calis fallina throuch the cascadeWhat you should decideGiven vour simplification (only Client::searcho raises RateLimitException, onlv MatchActivityCrmData middleware handles it, vou have two reasonableontions,Option A — Accept the gap, document it. The PR's scope is "handle search endpoint rate limits" since that's where 429s actually hurt. Email-by-id 429s are rareand currently treated as "not found" - same as before this PR. Nothing breaks, you just don't fix this particular gap. Add a TODO/follow-up ticketOntion B - Wrao_netContactByEmaillin executeRenest@_too.Five-line chance. then the email-match.sten_would also_throw.RateldimitExcention middlewarebecause those aren't on the MatchActivityCrmData hot path,Mwrecommendation.OptionB.tsiattinvaddiiion.that closes.the matchina cascade.stirststep.whichls.the.mostchitpath. withoutrt.theinew.midclleware missesthe most common entry point of MatchActivityCrmData's CRM lookupsD phpAsk anything (&AL)+ <> CodeClaude Opus 4.7 MediumHiahliaht wordsyXBB?9 differencesuse HubSpot\Client\ Crm\Pipelines Model\PipelineStage:use Hubspot Cllent Crm Properties Model Property:use HubSpot\ Discovery Discovery:Jiminny Exceptions CrmException:Jiminny\ Exceptions\RateLimitExceptionJaminnyExceotions SoclalAccountTokeninval1dzxcentionsuse Jiminny\Jobs\Crm\Note0bject;use Jiminny Models Crm Field:use Jiminny Services\Crm \BaseClient;nubiaie Function construct(SocialAccountService $socialAccountService,HubsnotPaginationSenvice SnaginationSenvico..HubsnotTokenManagen StokenManagen.parent::__construct($socialAccountService):thic-onaninationConvido = CnaainationConvido.Cthic-stalonMananon = CtakenManadon•WN Windsurf Toams 007-50UTF.8io 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15662
|
|
15661
|
Project: faVsco.js, menu
Firefox FileEditViewHisto Project: faVsco.js, menu
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жз• Support Daily - in 4h 53 mDOCKERDEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY]@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:07:30181screenpipe"O 86DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15661
|
|
15660
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15660
|
|
15659
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
PhostormVIewINavicareCodeKeractorFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitroledey© SyncRelatedActivityManager.php© BatchSyncCollectol© HubspotSyncStrategyBase.phpCachedcrmservicebecorator.onp© ProspectCache.phpe balchsynckealsseС Cпескапокetrукemotematch.ongo closedDealstagess)MatchactivitycrmData.ong© CrmActivityService.phgDealrielasservice.g© CrmObiectsResolver.phpc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements Hubspotcllentintertace— | A2 A65 X1X1AC) FieldT vpeconvertee Hubspotclientinter* doaram bool sarchivedc) Hubspotlokenman© PayloadBuilder.php* aneturn array<Owner>|[C) RemotecrmobiectrP ResponseNormalize13 usagesc) Service,onoC)SvncFieldAction.onpublic function getOwnersArchived(bool $archived = true): arrayC) SvncRelatedActivitSendpoint ="crm/vs/owners':C) WebhookSvncBatclSoueryParams =v MintearationAorarchived" = Sarchived ? "true': "false',M AcceccorsConfigSquervstrind = htto bulld auery squerv?arams):D DTOSowners = 1D FiltersD JobsProcnectSearchStratrySresnonse = Sthis->makeRequest(endnoint• Sendooint. auervStrina: SauervStrino)•W service lraltsSresnonselata = Sresnonse?->toArravO.© DataClient.phpConsole,LOg XChanaes 11 files= env.local aonTJ0 + → Side-by-side viewer •8 35f036ac app/Services/Crm/Hubspot/Client.phgDo not ignoreHiahliaht wordsyXBô ?© HandleHubspotRateLimit.php app/Jobs/Middleware© HubspotClientinterface.php app/Services/Crm/Hubspotpublic tunction -_constructSocialAccountService SsocialAccountService.i© HubspotPaginationService.php app/Services/Crm/Hubspot/Pagination@ JiminnyDebugCommand.php app/Console/Commandsphp logging.php config© MatchActivityCrmData.php app/Jobs/CrmHubspotTokenManager stokenanagennarent:: construct(SsocialAccountService):Sthis->naginationService = SnaainationService:© MatchCrmData.php app/Jobs/Activity/Import© PaginationState.php app/Services/Crm/Hubspot/PaginationSthis->tokenManaden = StokenManagen:© RateLimitException.php app/ExceptionsSthis->setRaselln1(self.•RASE URID:Unversioned Files 9 files=.env.nikilocal appE.env.other appSthis->setVension/self.•MTN APT VERSTON)•@) CanAccessAiReportsTest.php tests/Unit/Policies© CreateMockAskJiminnvReportResultCommand.php app/Console/Commands/Rep_public function getMinimumApiVersion: string6 favicon.ico publicnotunn colf.-MTN ApT VEpSTON.Eids.txt aprT raw sal querv.sal appataceneratoro nc) SimulateWe.hookscommand.ohoaoo/console/commands/crm/=uosootMJ WEBHOOK FILTERING IMPLEMENTATION.md apdSlastRecordIdolner Code will hoin INF to underctand vour Laravel ann code II Generate II Don't Show Anvmore (50 minutes aao)=custom.log~=laravel.logA SF (jiminny@localhost]4 HS_local (jiminny@localhost]# console [PKOb.A console [EU]# console [slAGiNg)[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn"Concent-lvoe". "apolicacionison.charser=utr-on"Transter-Encod1nq":"chunked")."Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncludeSubDomains: preload")n"Vary"*"orioin.accent-encodino""access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\","x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"So+-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtm0V-1778163675-1.0.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Report-To":["{"endpoints\":[{"unl".httns:a.nel.cloudflane.com./nenon+|W/v42c=NVA1cVTPQfVm32anS0axVF/sd2R\"group)":\"cf-nel\","max_age\":604800}"],"NEL":["{success_traccion.0.01rreportto. "cr-nel,• suppont Dally • In 4h 03m100% Lz. Mon 11 May 10:07:25AskJiminnyReportActivityServiceTestvCascadeHubspot Rate Limit Re+0 ..3. We just made an additional API call while already rate-limited - amplifying the pressure4. If matchßvPhone returns nothina too, the activity aets nersisted las "no CPM match" even though the contact eyists in HubSnot. Wrona data, no sianal to retrvThe middleware on MatchActivityCrmData cannot help, because the 429 was already swallowed before any RateLimitException could be raised.how serious is it reallv?In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the AP| add-on), which is much more generous than the search endpoint (5 RPS). 429s on this endpoint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail• When it does hannen. the cost is silent data corruntion wrona CRM match) olus extra wasted APl calis fallina throuch the cascadeWhat you should decideGiven vour simplification (only Client::searcho raises RateLimitException, onlv MatchActivityCrmData middleware handles it, vou have two reasonableontions,Option A — Accept the gap, document it. The PR's scope is "handle search endpoint rate limits" since that's where 429s actually hurt. Email-by-id 429s are rareand currently treated as "not found" - same as before this PR. Nothing breaks, you just don't fix this particular gap. Add a TODO/follow-up ticketOntion B - Wrao_netContactByEmaillin executeRenest@_too.Five-line chance. then the email-match.sten_would also_throw.RateldimitExcention middlewarebecause those aren't on the MatchActivityCrmData hot path,Mwrecommendation.OptionB.tsiattinvaddiiion.that closes.the matchina cascade.stirststep.whichls.the.mostchitpath. withoutrt.theinew.midclleware missesthe most common entry point of MatchActivityCrmData's CRM lookupsD phpAsk anything (&AL)+ <> CodeClaude Opus 4.7 MediumQ differencesCurrent versionpublic function -_constructlSocialAccountService SsocialAccountService.HubsootTokenManager StokenManagernarent:: construct(ssocialAccountService):$this->paginationService = $paginationService;Sthis->tokenManaden = StokenManagen:Sthis->setRaselln1(self••RASE |RI)•Sthis->setVension/self••MTN APT VERSTON).* Reacts to a rate limits (429) from HubSpot by translating it* into a RateLimitException carrying retry _after.* Wrap any outbound HubSpot call (SDK or raw HTTP) like:Sthis->executeRequest(fn • => Sthis->qetNewInstance0->crmО->...);WN Windsurf Teams007-50 UTF.8Po 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15659
|
|
15658
|
Project: faVsco.js, menu
PhostormVIewINavicareCode Project: faVsco.js, menu
PhostormVIewINavicareCodeKeractorFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitProletey© HubspotPaginationService.php© SyncRelatedActivityManager.php© BatchSyncCollectol© HubspotSyncStrategyBase.phpy syneermenttes tralt.onpCachedcrmservicebecorator.onp© ProspectCache.phpe balchsynckealsseС Cпескапокetrукemotematch.ongc clientonec closeaDealstagess @ MatchacuivitycrmData.ong© CrmActivityService.phpDealrielasservice.g© CrmObiectsResolver.phpc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements Hubspotcllentintertace— | A2 A65 X1X1AC) FieldT vpeconvertee Hubspotclientinter* doaram bool sarchivedc) Hubspotlokenman© PayloadBuilder.phpC) RemotecrmobiectrP ResponseNormalize* @neturn array<0wner>|[]13 usagesc) Service,onoC)SvncFieldAction.onpublic function get0wnersArchived(bool $archived = true): arrayC) SvncRelatedActivitSendpoint ="crm/vs/owners':C) WebhookSvncBatclSoueryParams =v MintearationAorarchived" = Sarchived ? "true': "false',M AcceccorsConfigDDTOD FiltersD JobsProcnectSearchStraSquervstrind = htto bulld auery squerv?arams):Sowners = 1trySresnonse = Sthis->makeRequest(endnoint• Sendooint. auervStrina: SauervStrino)•W service lralts© DataClient.phpSresnonselata = Sresnonse?->toArravO•Console,Log XChanaes 11 files= env.local aonTJ0 + → Side-by-side viewer •8 35f036ac app/Services/Crm/Hubspot/Client.phpDo not ignoreHiahliaht wordsyXBB ?© HandleHubspotRateLimit.php app/Jobs/Middleware© HubspotClientinterface.php app/Services/Crm/Hubspot© HubspotPaginationService.php app/Services/Crm/Hubspot/Pagination@ JiminnyDebugCommand.php app/Console/Commandsphp logging.php config© MatchActivityCrmData.php app/Jobs/Crm© MatchCrmData.php app/Jobs/Activity/Import© PaginationState.php app/Services/Crm/Hubspot/Pagination© RateLimitException.php app/ExceptionsUnversioned Files 9 files=.env.nikilocal appE.env.other app@) CanAccessAiReportsTest.php tests/Unit/Policies© CreateMockAskJiminnvReportResultCommand.php app/Console/Commands/Rep_6 favicon.ico publicEids.txt aprT raw sal querv.sal appSthis->setVersion(self:: "IN APT VERSTON"public function getMinimumApiVersion: stringneturn self:-MIN APT VERSTON.Clac+RocondTdi* Othrows DealApiException* dchrows CrmSxcentionpublic function getOpportunityById(string ScrmId, array $fields): arrayc) SimulateWe.hookscommand.ohoaoo/console/commands/crm/=uosootM. WEBHOOK FILTERING IMPLEMENTATION.md apptry{olner Code will hoin INF to underctand vour Laravel ann code II Generate II Don't Show Anvmore (50 minutes aao)S0 hill• suppont Dally • In 4h 03m100% C47 • Mon 11 May 10:07:22AskJiminnyReportActivityServiceTest v+0 ..=custom.log~=laravel.logA SF (jiminny@localhost]4 HS_local [jiminny@localhost]# console [PKOb.# console leu)# console [slAGiNg)[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn"concenc-lyoe. apolicacionison.charser=utt-on"Transter-Encod1nq":"chunked")."Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncludeSubDomains: preload")naccent-encodino""access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\","x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"So+-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtmoV-1778163675-1.0.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Report-To":["{"endpoints\":[{"unl".httns:la.nel.cloudflane.com./renon+W/v42c=NVA1cVTPQfVm32anS0#xVF/sd2Rl\"group)":\"cf-nel\","max_age\":604800}"],"NEL":["{success_traccion.0.01rreportto. "cr-nel,CascadeHubspot Rate Limit Re3. We just made an additional API call while already rate-limited - amplifying the pressure4.IfmatichByPhone.returns.nothina.too.the activitv.aets.nersistedlas. "no_CRM.match" even.thouah.the contact exists.in.HubSoot. Wrona data.no_sianalito.retrThe middleware on MatchActivityCrmData cannot help, because the 429 was already swallowed before any RateLimitException could be raisedhow serious is it reallv?In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the AP| add-on), which is much more generous than the search endpoint (5 RPS). 429s on this endpoint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail• When it does hannen. the cost is silent data corruntion wrona CRM match) olus extra wasted APl calis fallina throuch the cascadeWhat you should decideGiven vour simplification (only Client::searcho raises RateLimitException, onlv MatchActivityCrmData middleware handles it, vou have two reasonableontions,Option A — Accept the gap, document it. The PR's scope is "handle search endpoint rate limits" since that's where 429s actually hurt. Email-by-id 429s are rareand currently treated as "not found" - same as before this PR. Nothing breaks, you just don't fix this particular gap. Add a TODO/follow-up ticketOntion B - Wrao_netContactByEmaillin executeRenest@_too.Five-line chance. then the email-match.sten_would also_throw.RateldimitExcention middlewarebecause those aren't on the MatchActivityCrmData hot path,Mwrecommendation.OptionB.tsiattinvaddiiion.that closes.the matchina cascade.stirststep.whichls.the.mostchitpath. withoutrt.theinew.midclleware missesthe most common entry point of MatchActivityCrmData's CRM lookupsD phpAsk anything (&AL)+ <> CodeClaude Opus 4.7 MediumQ differencesCurront vorcionreturn 10:public function getMinimumApiVersion: stringneturn self.-MIN APT VERSTON:* Execute a search request against HubSpot CRM obiects with rate limiting.* apanam string SobiectType The obiect tupe ('deals', 'companies', 'contacts', 'calls')* apanam arrau<string, mixed> $payload The search payload with filters, sorts, properties, etc.* dreturn arrou Ine search response with "resulrs'* Athrows RateLimitException When rate limit is hitl"Total", 'naorno' keusWN Windsurf Toams 007-50UTF.8io 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15658
|
|
15657
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жз• Support Daily - in 4h 53 mDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:07:20181screenpipe"0 ₴6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15657
|
|
15656
|
PhpStormVIewINavicareCodeLaravelKeractorTOOISFV fa PhpStormVIewINavicareCodeLaravelKeractorTOOISFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitProiect v© BatchSyncCollectolyhuospotsyncstrategybase.ongCachedcrmservicebecorator.pnp© ProspectCache.phpe balchsynckealssec closeaDealstagess© MatchactivityermData.png© CrmActivityService.phgDealrielasservice.gc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements Hubspotcllentintertace— | A2 A65 X1X1AC) FieldT vpeconvertee Hubspotclientinter* doaram bool sarchivedc) Hubspotlokenman© PayloadBuilder.php* aneturn array<Owner>|[C) RemotecrmobiectrP ResponseNormalize13 usagesc) Service.onoC)SvncFieldAction.onpublic function getOwnersArchived(bool $archived = true): arrayC) SvncRelatedActivit© WebhookSyncBatcv MintearationAor9099101911Sendpoint ="crm/vs/owners':SoueryParams ==> $archived ? 'true' : 'false',M Acceccors913ConfigSquervstrind = htto bulld auery squerv?arams):D DTO• M Silters915Sowners = 1D Jobsh ProcnectSoarchStrtrySresnonse = Sthis->makeRequest(endnoint• Sendooint. auervStrina: SauervStrino):W service lraltsSresnonselata = Sresnonse?->toArravO.© DataClient.phpLocal ChangesConsole,Log xSide-bv-side viewerDo notignore Highlight words x 15 B?> Move non paginator search calls to client-›search 1 file, Moments ago> Almost ready one call to be tested 8 files, 8.05.26, 14:54› hubspot rate limit temp checkout 17 files, 8.05.26. 11:48mbase version aoo/services/crm/rubsoot/Service.ohoprivate const strina LOG DATE FORMAT = 'Y-m-d H:i:s'•› saved search delete 1 file. 27.04.26. 14:58private const int BATCH UPDATE LIM = 1001> expiration message 6 files, 27.04.26, 9:23private const string TEN_SECONDLY_ROLLING_POLICY = 'TEN_SECONDLY_ROLLING';> UP dev loas 1 file, 24.04.26. 12:02orivate const int TEN SECONDLY ROLLING LIME = 10:› tem test 1 file, 24.04.26, 10:36> user transformer 3 files. 23.04.26. 15:43> email ready 9 files, 23.04.26. 15:31> Email on not sent 5 files. 21.04.26. 19:38nrivate const strina TYPE NOTE = "NOTE!.> Planhat track interest event 2 files. 21.04.26. 13:56private const string TYPE_MEETING = 'MEETING' :> UP trackina controller 1 file. 20.04. 26. 9:22ninnv|Services|Cr1Hubsoot > Service > findo AAAADon't send if no s3 path (ix vodate to failed) 1 file. 17.04.26. 12:56.Sdata =Me> FEATURE CHANGES 3 files. 6.04. 26. 13:40web.oho fix router 1 fille. 1.04.26. 9:231• Sf Tune error fiy 2 files 19.26// Use the new V3 API to find contacts based on additional fields.foreach (['companies'.'contacts'] as SobiectType) {neyt automated renort tack 1 file 21.02.26 10.55Sobhecclype. searchhwho knows 2 fles 24.03 26 14:15Spayload = Sthis->qenerateNameSearchPayload(Sname, Soffset, Scount):temn fix 22 filos 24 02 26 10-51Stype = SobiectType === 'companies' ? 'account' : 'contact':.env local not sure why it was changed 1 file, 24.03.26, 10:04Al Renorts 21 files 19 03 26 18-07tryfCode will hoin INF to underctand vour Laravel ann code II Generate II Don't Show Anvmore (50 minutes aaol• suppont Dally • In 4h 03m100% C47 • Mon 11 May 10:07:19AskJiminnyReportActivityServiceTestv= custom.log X=laravel.logA SF (jiminny@localhost]4 HS_local [jiminny@localhost]Cascade# console [PKob.# console [euJ# console [slAGiNg)Hubspot Rate Limit Re+0 ..[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {W19лV3. We just made an additional API call while already rate-limited - amplifying the pressure"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn4. If matchByPhone returns nothing too, the activity gets persisted as "no CRM match" even though the contact exists in Hubspot. wrong data, no signal to retryThe middleware on MatchActivityCrmData cannot help, because the 429 was already swallowed before any RateLimitException could be raised."Transter-Encod1nq":"chunked").how serious is it reallv?"Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the AP| add-on), which is much more generous than the search endpoint (5 RPS). 429s on this endpoint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncLudeSubDomains: preload")n• When it does hannen. the cost is silent data corruntion wrona CRM match) olus extra wasted APl calis fallina throuch the cascadeWhat you should decide"access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",Given vour simplification (only Client::searcho raises RateLimitException, onlv MatchActivityCrmData middleware handles it, vou have two reasonableontions,Option A — Accept the gap, document it. The PR's scope is "handle search endpoint rate limits" since that's where 429s actually hurt. Email-by-id 429s are rareand currently treated as "not found" — same as before this PR. Nothing breaks, you just don't fix this particular gap. Add a TODO/follow-up ticketOntion B - Wrao_netContactByEmaillin executeRenest@_too.Five-line chance. then the email-match.sten_would also_throw.RateldimitExcention middleware"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"Sot-Cookip"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtmoV-1778163675-1.0.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Renont-To"."","endpoints\":[{because those aren't on the MatchActivityCrmData hot path,Mwrecommendation.OptionB.tsiattinvaddiiion.that closes.the matchina cascade.stirststep.whichls.the.mostchitpath. withoutrt.theinew.midclleware missesthe most common entry point of MatchActivityCrmData's CRM lookups"unl".httns:a.nel.cloudflane.com./nenon+|W/v42c=NVA1cVTPQfVm32anS0axVF/sd2R\"group)":\"cf-nel\","max_age\":604800}"],"NEL":["{D phpAsk anything (&AL)success_traccion.0.01r+ ‹› CodeClaude Opus 4.7 Mediumreportto. "cr-nel,12 differences Shelved versionprivate const string LOG DATE FORMAT = 'Y-m-d H:i:s'-orivate const int BATCH UPDATE LTM = 1001private const string TEN_SECONDLY_ROLLING_POLICY = 'TEN_SECONDLY_ROLLING' :private const int TEN_SECONDLY_ROLLING_LIMIT = 10;107107private const string TYPE_NOTE = 'NOTE';111private const string TYPE_MEETING = 'MEETING';Sdata =Mo// Use the new V3 API to find contacts based on additional fields.foreach (['companies', 'contacts'] as $objectType) {Spayload = Sthis->generateNameSearchPayload($name, Soffset, $count):Stype = $obiectType === 'companies' ? 'account' : 'contact':tryfSresponse = Sthis->client->search(SobiectType, Spayload):W Windsurf Teams 907:59 UTF-8 P 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15656
|
|
15655
|
Firefox FileEditViewHistoryBookmarksProfilesToolsW Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жз• Support Daily - in 4h 53 mDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:07:19181screenpipe"0 ₴6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15655
|
|
15654
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
PhpStormVIewINavicareCodeLaravelKeractorFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitroledey© BatchSyncCollectolyhuospotsyncstrategybase.ongCachedcrmservicebecorator.pnp© ProspectCache.phpe balchsynckealssec closeaDealstagess© MatchactivityermData.png© CrmActivityService.phgDealrielasservice.gc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements HubspotclientintertaceM |A2 A65 X1X1AC) FieldT vpeconvertee Hubspotclientinterpublic function ensureValidTokenO: voidc) Hubspotlokenman880© PayloadBuilder.phpC) RemotecrmobiectrP ResponseNormalizeif (Sthis->oauthAccount === null) {c) Service,onoC)SvncFieldAction.onC) SvncRelatedActivitC) WebhookSvncBatclSnewlloken=sthis->tokenmanagen->ensureva.1dlokencsth1s->oauthaccountif (SnewToken |== null) iSthis->accesstoken = Snewtoken:v MintearationAorM Acceccors889896• D Config891D DTO> M Silterspublic function getConfigoreturn $this->config;D Jobsh ProcnectSoarchStrW service lralts© DataClient.php/ returns only active (archived=false)9 usageslocal ChangesConsole,Log xSide-bv-side viewer vDo not ignoreHighlight words -] x 13 ?› Move non paginator search calls to client-›search 1 file, Moments ago> Almost ready one call to be tested 8 files, 8.05.26, 14:54› hubspot rate limit temp checkout 17 files, 8.05.26. 11:48› saved search delete 1 file. 27.04.26. 14:58> expiration message 6 files, 27.04.26, 9:23> UP dev loas 1 file, 24.04.26. 12:02› tem test 1 file, 24.04.26, 10:36> user transformer 3 files. 23.04.26. 15:43> email ready 9 files, 23.04.26. 15:31> Email on not sent 5 files. 21.04.26. 19:38> Planhat track interest event 2 files. 21.04.26. 13:56> UP trackina controller 1 file. 20.04. 26. 9:22m base version aoo/stprivate const strina LOG DATE FORMAT = 'Y-m-d H:i:s'•private const int BATCH UPDATE LI = 1001private const string TEN_SECONDLY_ROLLING_POLICY = 'TEN_SECONDLY_ROLLING' :orivate const int TEN SECONDLY ROLLING LIMTT = 10:nrivate const strina TYPE NOTE = "NOTE!.private const string TYPE_MEETING = 'MEETING' :Don't send if no s3 path (fix vodate to failed) 1 file. 17.04.26. 12:56.> FEATURE CHANGES 3 files. 6.04. 26. 13:40Sdata =Meweb.oho fix router 1 fille. 1.04.26. 9:231• Sf Tune error fiy 2 files 19.26// Use the new V3 API to find contacts based on additional fields.foreach (['companies'.'contacts'] as SobiectType) {neyt automated renort tack 1 file 21.02 26 10•551who knows 2 files 24.02. 26 14:45temn fix 22 filec 24 02 26 10-51Sobhecclype searchSpayload = Sthis->qenerateNameSearchPayload($name, Soffset, Scount):Stype = SobiectType === 'companies' ? 'account' : 'contact':.env local not sure why it was changed 1 file, 24.03.26, 10:04> Al Renorts 21 filec 19 03 26 18:07tryf• suppont Dally • In 4h 03m100% C47 • Mon 11 May 10:07:16AskJiminnyReportActivityServiceTest v=custom.log ^=laravel.logA SF (jiminny@localhost]4 HS_local [jiminny@localhost]Cascade# console [PKOb.# console leu)# console [slAGiNG)Hubspot Rate Limit Re+0 ..[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {W19лV"neaders".?• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail.• When it does happen, the cost is silent data corruption (wrong CRM match) plus extra wasted API calls falling through the cascade"Uace":L"Inu,or May 2020 14.21.15 6Ml"JnWhat vou should decide"Transter-Encod1nq":"chunked")."Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],Given your simplification (only Client::search() raises RateLimitException, only MatchActivityCrmData middleware handles it), you have two reasonable"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncLudeSubDomains: preload")nOption A - Accept the gap, document it. The PR's scope is "handle search endpoint rate limits" since that's where 429s actually hurt. Email-by-id 429s are rareand currently treated as "not found" — same as before this PR. Nothina breaks, vou just don't tix this particular aap. Add a TODO/follow-up ticketOption B - Wrap getContactByEmail in executeRequest() too. Five-line change, then the email-match step would also throw RateLimitException → middlewarereleases the job. This is cheap and removes the silent-corruption risk for that one method. You don't need to touch getContactByld/getopportunityByld/etc.because those aren t on the Macchaccivicycrbata not patn.Mv.recommendation:Option_B.It's a tiny.addition.that closes the matchina_cascadels.first steo.which is.the most-hit path.Without it.the new.middleware misses"server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",Sketch,• php"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"Sot-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtmoV-1778163675-1010.1pustzc tuieazon getconttettytia trstratig seidlly array srietus ta ulr arra07-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Report-To":["{"endpoints\":[{try{"unl".httns:la.nel.cloudflane.com./renon+W/v42c=NVA1cVTPQfVm32anS0#xVF/sd2RNScontact = Sth1s→>executeRequest(fn 0 =s Sthis->aetNewinstance@->crmo=>contacts@-shasicAnol->getById($email, implode(',', $fields), null, false, 'email')\"group)":\"cf-nel\","max_age\":604800}"],"NEL":["{Ask anything (&AL)success_traccion.0.01r+ <> CodeClaude Opus 4.7 Mediumreportto. "cr-nel,12 differences Shelved versionprivate const string LOG DATE FORMAT = 'Y-m-d H:i:s'-orivate const int BATCH UPDATE LTM = 1001private const string TEN_SECONDLY_ROLLING_POLICY = 'TEN_SECONDLY_ROLLING' :private const int TEN_SECONDLY_ROLLING_LIMIT = 10;107107nrivate const strind TYPE NOTE & "NOTE!.nnivate const string TYpE MESTING = "MESTINGI•111Sdata =Me593// Use the new V3 API to find contacts based on additional fields.foreach (['companies', 'contacts'] as $objectType) {Spayload = Sthis->generateNameSearchPayload($name, Soffset, $count):Stype = SobiectType === 'companies' ? 'account' : 'contact':tryfSresponse = Sthis->client->search(SobiectTypeSpayload):WN Windsurf Toams 895•62UTF.8Po 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15654
|
|
15653
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• *з.• Support Daily - in 4h 53 mDOCKER881DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY]@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% <78• Mon 11 May 10:07:16181screenpipe™#6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15653
|
|
15652
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
PhostormVIewINavicareCodeKeractorWindowFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitroledey© HubspotPaginationService.php© BatchSyncCollectolC HubspotSyncStrategyBase.pngCachedcrmservicebecorator.onp© ProspectCache.phpe balchsynckealsseС Cпескапокetrукemotematch.ongc clientone© ClosedDealStagesS © MatchActivityCrmData.php© CrmActivityService.phgDealrilelasservice.c© CrmObiectsResolver.phpc)Decorateacuivilv.or© FieldDefinitions.phrclass Elzent extends Baseulzent implements hubspotelzentintertace42 465 M1 M1AC) FieldT vpeconvertee Hubspotclientinterpublic function ensureValidTokenO: voidc) HubspotlokenmanC) PavloadBuilder.phpif (Sthis->oauthAccount === nulb) {C) RemotecrmobiectP ResponseNormalizec) Service,onoC)SvncFieldAction.onC) SvncRelatedActivitC) WebhookSvncBatclSnewlloken=sthis->tokenmanagen->ensureva.1dlokencsth1s->oauthaccountif (SnewToken !== null) {Sthis->accesstoken = Snewtoken:v MintearationAorM Acceccors> D ConfigD DTO> D FiltersJobs• M ProcnectSearchStr.891public function getConfigoreturn $this->config;M ServiceTraits/I returns only active (archived=false)© DataClient.php9 usagesLocal ChangesConcole yLog X40• suppont Dally • In 4h 03m100% Lz• Mon 11 May 10:07:11AskJiminnyReportActivityServiceTest v+0 ..A SF (jiminny@localhost]4 HS_local (jiminny@localhost]# console [PKob.# console leu)# console [slAGiNg)[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn"concenc-lyoe. apolicacionison.charser=utt-on"Transter-Encod1nq":"chunked")"Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],"Strict-Transport-Security":["max-aqe=31536000: includeSubDomains: preload"].accent-encodino""access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\","x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"Sot-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtmoV-1778163675-1010.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Renont-To".r"*"endnoints"."unl".httns:la.nel.cloudflane.com./renon+W/v42c=NVA1cVTPQfVm32anS0#xVF/sd2RN"group\":\"cf-nel\","max_age\":604800}"],"NEL":["{success_traccion.0.01rreportto. "cr-nel,CascadeHubspot Rate Limit ReClient.php:516-523}catch (ContactApiException $e) ‹Sthis->log→>info('(Hubspot] Failed to fetch contact', ['email" => Semail,'reason' => $e->getMessage(),return [];It returns [). From the job's point of view, that's identical to "this email is not in HubSpot". So1. matchExactlyByEmail silently swallows the 429.2. The cascade falls through to matchByPhone (another HubSpot API call, this time the search endpoint).3. We just made an additional API call while already rate-limited — amplifvina the pressure.4. If matchByPhone returns nothing too, the activity gets persisted as "no CRM match" even though the contact exists in HubSpot. Wrong data, no signal to retryThe middleware on MatchAct ivitvCrmdata cannot helo, because the 429 was already swallowed before anv RateLimitExcention could be raisedHow serious is it reallv?In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the APl add-on). which is much more aenerous than the search endooint (5 RPS). 429s on this endooint are uncommon. But:• Durina heavy backfills they do happen — the same worker pool that's hammerina search will also be hammering getContactByEma i1.Ask anything (&AL)+ <> CodeClaude Opus 4.7 Medium+0|+→?No differencesW Windsurf Teams 885:63 UTF-8 P 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15652
|
|
15651
|
Diff
Changelist:
Changelist:
Changes
Show Diff
Fir Diff
Changelist:
Changelist:
Changes
Show Diff
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жз• Support Daily - in 4h 53 mDOCKER₴1DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY]@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:07:091881screenpipe"O 86DEV...
|
PhpStorm
|
Shelve Changes
|
NULL
|
15651
|
|
15650
|
Diff
Changelist:
Changelist:
Changes
Show Diff
Rol Diff
Changelist:
Changelist:
Changes
Show Diff
Rollback...
Refresh
Group By
Expand All
Collapse All
12 files, folder partially checked
app 10 files, folder partially checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
config 1 file, folder not checked...
|
PhpStorm
|
Shelve Changes
|
NULL
|
15650
|
|
15649
|
Diff
Changelist:
Changelist:
Changes
Show Diff
Rol Diff
Changelist:
Changelist:
Changes
Show Diff
Rollback...
Refresh
Group By
Expand All
Collapse All
12 files, folder partially checked
app 10 files, folder partially checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
config 1 file, folder not checked
.env.local not checked
app 10 files, folder partially checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
config 1 file, folder not checked
.env.local not checked
1 modified
Almost ready one call to be tested
Commit Message
Commit Message History
Previous Difference
Next Difference
Jump to Source
Compare Previous File
Compare Next File
Go to Changed File…
Side-by-side viewer
Do not ignore
Highlight words
Collapse Unchanged Fragments
Synchronize Scrolling
Disable Editing
Settings...
|
PhpStorm
|
Shelve Changes
|
NULL
|
15649
|
|
15648
|
Diff
Changelist:
Changelist:
Changes
Show Diff
Rol Diff
Changelist:
Changelist:
Changes
Show Diff
Rollback...
Refresh
Group By
Expand All
Collapse All
12 files, folder partially checked
app 10 files, folder partially checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
config 1 file, folder not checked
.env.local not checked
app 10 files, folder partially checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
Console/Commands 1 file, folder not checked
Exceptions 1 file, folder not checked
Jobs 3 files, folder not checked
Services/Crm/Hubspot 5 files, folder partially checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
Pagination 2 files, folder not checked
Client.php, class not checked
HubspotClientInterface.php, interface not checked
Service.php, class checked
config 1 file, folder not checked
.env.local not checked
1 modified
Almost ready one call to be tested
Commit Message
Commit Message History
Previous Difference
Next Difference
Jump to Source
Compare Previous File
Compare Next File
Go to Changed File…
Side-by-side viewer
Do not ignore
Highlight words
Collapse Unchanged Fragments...
|
PhpStorm
|
Shelve Changes
|
NULL
|
15648
|
|
15647
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15647
|
|
15646
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15646
|
|
15645
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15645
|
|
15644
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15644
|
|
15643
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:
Hide
19
Previous Highlighted Error
Next Highlighted Error
[2026-05-07 14:21:15] local.INFO: [Hubspot] DEBUG Getting headers {
"headers":{
"Date":["Thu,07 May 2026 14:21:15 GMT"],
"Content-Type":["application/json;charset=utf-8"],
"Transfer-Encoding":["chunked"],
"Connection":["keep-alive"],
"CF-Ray":["9f80deb8db60dc3a-SOF"],
"CF-Cache-Status":["DYNAMIC"],
"Strict-Transport-Security":["max-age=31536000; includeSubDomains; preload"],
"Vary":["origin,
accept-encoding"],
"access-control-allow-credentials":["false"],
"server-timing":["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",
cfr;desc=\"9f80deb8e7c6dc3a-IAD\""],
"x-content-type-options":["nosniff"],
"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],
"Set-Cookie":["__cf_bm=SIUrtdQgXVrik50pdqF6hZVYKhzTnQBidvMabeCtm0Y-1778163675-[IP_ADDRESS]-rI.ZggtDKxTge5zr8_2gbBfWMQQ.ufZEXDZyHz2mBUFdzdo2gTHEsOkXMSEShjK0hGYxNhUGM1ZoBpX7BcFZcHEjA7Cs_.SMUhUnd2nYjko; path=/; expires=Thu,
07-May-26 14:51:15 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None"],
"Report-To":["{
\"endpoints\":[{
\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=NYAlsVTP0fYm32qrSDjxYE4sd2RWRqiSp3wHsmdEgZlzoYdxI%2BIxVpHmsKn3O%2BKVA3mFIJ2m7YRECDGSM%2BW2IYTzo6FM4%2BdUIjURO8srzKSvJgZ%2BQ6R79arKQw3uHLlX\"}],
\"group\":\"cf-nel\",
\"max_age\":604800}"],
"NEL":["{
\"success_fraction\":0.01,
\"report_to\":\"cf-nel\",
\"max_age\":604800}"],
"Server":["cloudflare"]}} {
"correlation_id":"95236535-ec98-4541-b92a-adfa73b69eab",
"trace_id":"c7ab8365-903f-46d4-9403-0e5b551e3545"}
Project
Project
New File or Directory…
Expand Selected
Collapse All
Options
Hide...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15643
|
|
15642
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Start Listening for PHP Debug Connections
AskJiminnyReportActivityServiceTest
Run 'AskJiminnyReportActivityServiceTest'
Debug 'AskJiminnyReportActivityServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Sync Changes
Hide This Notification
Code changed:
Hide
2
65
1
1
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot;
use HubSpot\Client\Crm\Deals\ApiException as DealApiException;
use HubSpot\Client\Crm\Contacts\ApiException as ContactApiException;
use HubSpot\Client\Crm\Companies\ApiException as CompanyApiException;
use HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectWithAssociations as ContactsWithAssociations;
use HubSpot\Client\Crm\Companies\Model\SimplePublicObjectWithAssociations as CompaniesWithAssociations;
use HubSpot\Client\Crm\Deals\Model\SimplePublicObjectWithAssociations as DealWithAssociations;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectInput;
use HubSpot\Client\Crm\Objects\Model\SimplePublicObjectWithAssociations as ObjectWithAssociations;
use HubSpot\Client\Crm\Pipelines\Model\Error;
use HubSpot\Client\Crm\Pipelines\Model\PipelineStage;
use HubSpot\Client\Crm\Properties\Model\Property;
use HubSpot\Discovery\Discovery;
use Jiminny\Exceptions\CrmException;
use Jiminny\Exceptions\RateLimitException;
use Jiminny\Exceptions\SocialAccountTokenInvalidException;
use Jiminny\Jobs\Crm\NoteObject;
use Jiminny\Models\Crm\Field;
use Jiminny\Services\Crm\BaseClient;
use Jiminny\Services\Crm\Hubspot\DTO\Response\Owner;
use Jiminny\Services\SocialAccountService;
use SevenShores\Hubspot\Exceptions\BadRequest;
use SevenShores\Hubspot\Exceptions\HubspotException;
use SevenShores\Hubspot\Factory;
use SevenShores\Hubspot\Http\Response;
use Jiminny\Services\Crm\Hubspot\Pagination\HubspotPaginationService;
use Throwable;
/**
* @phpstan-type CrmFieldOption array{id:string, label:string, value?:string}
*/
class Client extends BaseClient implements HubspotClientInterface
{
public const string MIN_API_VERSION = '2';
public const string BASE_URL = '[URL_WITH_CREDENTIALS] T
* @param callable(): T $apiCall
* @return T
*
* @throws RateLimitException
*/
private function executeRequest(callable $apiCall)
{
try {
return $apiCall();
} catch (Throwable $e) {
if ($this->isHubspotRateLimit($e)) {
$retryAfter = $this->parseRetryAfter($e);
$this->log->warning('[Hubspot] Received 429 from API', [
'team_id' => $this->config->team_id,
'config_id' => $this->config->getId(),
'retry_after' => $retryAfter,
'reason' => $e->getMessage(),
]);
throw new RateLimitException('Hubspot returned 429', $retryAfter, $e);
}
throw $e;
}
}
public function isHubspotRateLimit(Throwable $e): bool
{
if ($e instanceof BadRequest
|| $e instanceof DealApiException
|| $e instanceof ContactApiException
|| $e instanceof CompanyApiException
|| $e instanceof \GuzzleHttp\Exception\RequestException
) {
return (int) $e->getCode() === 429;
}
return false;
}
public function parseRetryAfter(Throwable $e): int
{
// First try to get Retry-After from response headers
if (method_exists($e, 'getResponseHeaders')) {
$headers = $e->getResponseHeaders() ?: [];
$value = $headers['Retry-After'] ?? $headers['retry-after'] ?? null;
if (is_array($value)) {
$value = $value[0] ?? null;
}
if (is_numeric($value)) {
return (int) $value;
}
}
if (method_exists($e, 'getResponseBody')) {
$body = $e->getResponseBody();
if (is_string($body)) {
$body = json_decode($body, true) ?? [];
}
$policyName = $body['policyName'] ?? $body['policy'] ?? $body['context']['policyName'] ?? null;
if ($policyName === 'TEN_SECONDLY_ROLLING' || $policyName === 'ten_secondly_rolling') {
return 10;
}
if ($policyName === 'SECONDLY' || $policyName === 'secondly') {
return 1;
}
}
$this->log->warning('[Hubspot] No retry-after header or policy name found, using default', [
'exception_class' => get_class($e),
]);
return 10;
}
public function getMinimumApiVersion(): string
{
return self::MIN_API_VERSION;
}
public function getInstance(): Factory
{
return new Factory([
'key' => $this->accessToken,
'oauth2' => true,
'base_url' => $this->baseUrl,
]);
}
public function getNewInstance(): Discovery
{
return \HubSpot\Factory::createWithAccessToken($this->accessToken);
}
/**
* Secondly and daily limits for Hubspot API
*
* Product Tier: Free & Starter | Professional & Enterprise | API add-on (any tier)
* Burst: 100/10 seconds | 150/10 seconds | 200/10 seconds
* Daily: 250,000 | 500,000 | 1,000,000
*
* Official documentation states: The search endpoints are rate limited to five requests per second.
* Since with 5 RPS were still hitting secondly rate limits we lowered it to 4
*/
public function getPaginatedData(array $payload, string $type, int $offset = 0): array
{
$total = 0;
$lastId = null;
$rows = [];
foreach ($this->getPaginatedDataGenerator($payload, $type, $offset, $total, $lastId) as $row) {
$rows[] = $row;
}
return ['results' => $rows, 'total' => $total, 'last_record' => $lastId];
}
/**
* @throws HubspotException
* @throws SocialAccountTokenInvalidException
* @throws BadRequest
*/
public function getPaginatedDataGenerator(
array $payload,
string $type,
int $offset = 0,
int &$total = 0,
?string &$lastRecordId = null
): \Generator {
return $this->paginationService->getPaginatedDataGenerator(
$this,
$payload,
$type,
$offset,
$total,
$lastRecordId
);
}
/**
* Execute a search request against HubSpot CRM objects with rate limiting.
*
* @param string $objectType The object type ('deals', 'companies', 'contacts', 'calls')
* @param array<string, mixed> $payload The search payload with filters, sorts, properties, etc.
* @return array The search response with 'results', 'total', 'paging' keys
* @throws RateLimitException When rate limit is hit
* @throws HubspotException On API errors
*/
public function search(string $objectType, array $payload): array
{
$endpoint = self::BASE_URL . "/crm/v3/objects/{$objectType}/search";
return $this->executeRequest(function () use ($endpoint, $payload) {
$response = $this->getInstance()->getClient()->request('POST', $endpoint, ['json' => $payload]);
return $response->toArray();
});
}
/**
* @throws DealApiException
* @throws CrmException
*/
public function getOpportunityById(string $crmId, array $fields): array
{
try {
// $deal = $this->executeRequest(fn () => $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$deal = $this->getNewInstance()->crm()->deals()->basicApi()->getById(
$crmId,
implode(',', $fields),
'companies,contacts'
);
} catch (DealApiException $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $deal instanceof DealWithAssociations) {
throw new CrmException('Deal not found');
}
return [
'id' => $deal->getId(),
'properties' => $deal->getProperties(),
'associations' => $deal->getAssociations(),
];
}
/**
* Generic batch read method for HubSpot objects
*
* @param string $objectType The object type ('deals', 'companies', 'contacts')
* @param array<string> $crmIds Array of HubSpot object IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with object data
*/
private function batchReadObjects(string $objectType, array $crmIds, array $fields): array
{
if (empty($crmIds)) {
return [];
}
$this->validateBatchSize($objectType, $crmIds);
$this->ensureValidToken();
try {
$batchConfig = $this->createBatchConfiguration($objectType);
$batchReadRequest = $this->prepareBatchRequest($batchConfig, $crmIds, $fields);
$response = $batchConfig['api']->read($batchReadRequest);
$this->validateApiResponse($response, $objectType);
$results = $this->processApiResults($response);
$this->logBatchResults($objectType, $crmIds, $results);
return $results;
} catch (\Throwable $e) {
$this->handleBatchError($e, $objectType, $crmIds);
}
}
private function validateBatchSize(string $objectType, array $crmIds): void
{
if (count($crmIds) > 100) {
throw new \InvalidArgumentException("Batch size cannot exceed 100 {$objectType}");
}
}
private function createBatchConfiguration(string $objectType): array
{
$configurations = [
'deals' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Deals\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Deals\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->deals()->batchApi(),
],
'companies' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Companies\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Companies\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->companies()->batchApi(),
],
'contacts' => [
'batchReadRequest' => new \HubSpot\Client\Crm\Contacts\Model\BatchReadInputSimplePublicObjectId(),
'inputClass' => \HubSpot\Client\Crm\Contacts\Model\SimplePublicObjectId::class,
'api' => $this->getNewInstance()->crm()->contacts()->batchApi(),
],
];
if (! isset($configurations[$objectType])) {
throw new \InvalidArgumentException("Unsupported object type: {$objectType}");
}
return $configurations[$objectType];
}
private function prepareBatchRequest(array $batchConfig, array $crmIds, array $fields): object
{
$batchReadRequest = $batchConfig['batchReadRequest'];
$inputClass = $batchConfig['inputClass'];
$inputs = array_map(function ($crmId) use ($inputClass) {
$input = new $inputClass();
$input->setId($crmId);
return $input;
}, $crmIds);
$batchReadRequest->setInputs($inputs);
$batchReadRequest->setProperties($fields);
return $batchReadRequest;
}
private function validateApiResponse($response, string $objectType): void
{
if (! $response) {
throw new CrmException("HubSpot API returned null response for {$objectType} batch read");
}
}
private function processApiResults($response): array
{
$results = [];
$responseResults = $response->getResults();
if ($responseResults) {
foreach ($responseResults as $object) {
if ($object && $object->getId()) {
$results[$object->getId()] = [
'id' => $object->getId(),
'properties' => $object->getProperties() ?: [],
];
}
}
}
return $results;
}
private function logBatchResults(string $objectType, array $crmIds, array $results): void
{
$this->log->info("[HubSpot] Batch fetched {$objectType}", [
'requested_count' => count($crmIds),
'returned_count' => count($results),
'crm_ids' => $crmIds,
]);
}
private function handleBatchError(\Throwable $e, string $objectType, array $crmIds): void
{
$errorMessage = $e->getMessage() ?: 'Unknown error';
$errorTrace = $e->getTraceAsString() ?: 'No trace available';
$this->log->error("[HubSpot] Failed to batch fetch {$objectType}", [
'crm_ids' => $crmIds,
'error' => $errorMessage,
'trace' => $errorTrace,
]);
throw new CrmException("Failed to batch fetch {$objectType}: " . $errorMessage);
}
/**
* Batch read multiple opportunities by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot deal IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with opportunity data
*/
public function getOpportunitiesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('deals', $crmIds, $fields);
}
/**
* Batch read multiple companies by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot company IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with company data
*/
public function getCompaniesByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('companies', $crmIds, $fields);
}
/**
* Batch read multiple contacts by their CRM IDs
*
* @param array<string> $crmIds Array of HubSpot contact IDs (max 100)
* @param array<string> $fields Array of property names to fetch
*
* @return array<string, array> Array keyed by CRM ID with contact data
*/
public function getContactsByIds(array $crmIds, array $fields): array
{
return $this->batchReadObjects('contacts', $crmIds, $fields);
}
/**
* @throws CompanyApiException
* @throws CrmException
*/
public function getAccountById(string $crmId, array $fields): array
{
try {
$company = $this->getNewInstance()->crm()->companies()->basicApi()->getById(
$crmId,
implode(',', $fields),
);
} catch (CompanyApiException $e) {
$this->log->info('[Hubspot] Failed to fetch account', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $company instanceof CompaniesWithAssociations) {
throw new CrmException('Account not found');
}
return [
'id' => $company->getId(),
'properties' => $company->getProperties(),
];
}
/**
* @throws ContactApiException
* @throws CrmException
*/
public function getContactById(string $crmId, array $fields): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$crmId,
implode(',', $fields)
);
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'crm_id' => $crmId,
'reason' => $e->getMessage(),
]);
throw $e;
}
if (! $contact instanceof ContactsWithAssociations) {
throw new CrmException('Contact not found');
}
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
}
/**
* This is email search request that Hubspot offers as GET (more generous quota)
*/
public function getContactByEmail(string $email, array $fields = []): array
{
try {
$contact = $this->getNewInstance()->crm()->contacts()->basicApi()->getById(
$email,
implode(',', $fields),
null,
false,
'email'
);
return [
'id' => $contact->getId(),
'properties' => $contact->getProperties(),
];
} catch (ContactApiException $e) {
$this->log->info('[Hubspot] Failed to fetch contact', [
'email' => $email,
'reason' => $e->getMessage(),
]);
return [];
}
}
/**
* @throws CrmException
*/
public function fetchProperty(string $objectType, string $propertyId): Property
{
$result = $this->getNewInstance()->crm()->properties()->coreApi()->getByName($objectType, $propertyId);
if (! $result instanceof Property) {
$this->log->error('[Hubspot] Failed to fetch property', [
'object_type' => $objectType,
'property_id' => $propertyId,
'reason' => $result->getMessage(),
]);
throw new CrmException('Failed to fetch property');
}
return $result;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchPropertyOptions(string $objectType, string $propertyId): array
{
/** @var array<CrmFieldOption> */
return $this->fetchProperty($objectType, $propertyId)->getOptions();
}
/**
* @return array<array{id:string, label:string, deleted:bool}>
*/
public function fetchCallDispositions(): array
{
/** @var Response $response */
$response = $this->getInstance()->engagements()->getCallDispositions();
/**
* @var array<array{
* id:string,
* label:string,
* deleted: bool
* }>
*/
return $response->toArray();
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityPipelineStages(): array
{
$stages = [];
$apiResponse = $this->getNewInstance()->crm()->pipelines()->pipelinesApi()->getAll('deals');
if ($apiResponse instanceof Error) {
$this->log->error('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $apiResponse->getMessage(),
]);
return [];
}
foreach ($apiResponse->getResults() as $pipeline) {
$pipelineStages = array_map(
static function (PipelineStage $stage) {
return [
'id' => $stage->getId(),
'label' => $stage->getLabel(),
];
},
$pipeline->getStages()
);
$stages = array_merge($stages, $pipelineStages);
}
return $stages;
}
public function fetchOpportunityPipelines(): array
{
$pipelines = [];
try {
$apiResponse = $this->makeRequest('/crm/v3/pipelines/deals');
} catch (\Exception $e) {
$this->log->info('[Hubspot] Failed to fetch opportunity pipelines', [
'reason' => $e->getMessage(),
]);
return [];
}
$response = $apiResponse->toArray();
foreach ($response['results'] as $pipeline) {
$pipelines[] = [
'id' => $pipeline['id'],
'label' => $pipeline['label'],
];
}
return $pipelines;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchMeetingOutcomeFieldOptions(Field $field): array
{
return $field->getCrmProviderId() === 'meetingOutcome'
? $this->fetchMeetingOutcomeTypes()
: $this->fetchCallActivityTypes();
}
public function fetchMeetingOutcomeTypes(): array
{
return $this->extractMeetingTypeOptions(
'[URL_WITH_CREDENTIALS] Response $response */
$response = $this->getInstance()
->getClient()
->request('GET', $endpoint);
/**
* @var array<array{
* value: string,
* label: string,
* displayOrder: int
* }> $optionData
*/
$optionData = $response->toArray()['options'] ?? [];
$options = [];
foreach ($optionData as $item) {
$options[] = [
'id' => $item['value'],
'value' => $item['value'],
'label' => $item['label'],
'display_order' => $item['displayOrder'],
];
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchDispositionFieldOptions(): array
{
$options = [];
$dispositions = $this->fetchCallDispositions();
foreach ($dispositions as $disposition) {
if ($disposition['deleted'] !== false) {
continue;
}
$option['value'] = $disposition['id'];
$option['id'] = $disposition['id'];
$option['label'] = $disposition['label'];
$options[] = $option;
}
return $options;
}
/**
* @return array<CrmFieldOption>
*/
public function fetchOpportunityFieldOptions(Field $field): array
{
if ($field->isStageField()) {
return $this->fetchOpportunityPipelineStages();
}
if ($field->isPipelineField()) {
return $this->fetchOpportunityPipelines();
}
return $this->fetchPropertyOptions('deals', $field->getCrmProviderId());
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function makeRequest(string $endpoint, $method = 'GET', $payload = [], ?string $queryString = null)
{
$endpoint = self::BASE_URL . $endpoint;
if ($method === 'GET') {
$response = $this->getInstance()->getClient()?->request(
method: $method,
endpoint: $endpoint,
query_string: $queryString
);
} else {
$response = $this->getInstance()->getClient()->request($method, $endpoint, [
'json' => ($payload),
]);
}
//
// $max = $response->getHeaderLine('X-HubSpot-RateLimit-Max'); // "110"
// $remaining = $response->getHeaderLine('X-HubSpot-RateLimit-Remaining'); // "109"
// $interval = $response->getHeaderLine('X-HubSpot-RateLimit-Interval-Milliseconds'); // "10000"
// $body = json_decode((string) $response->getBody(), true);
//
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$max ' . PHP_EOL . print_r($max, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$remaining ' . PHP_EOL . print_r($remaining, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$interval ' . PHP_EOL . print_r($interval, true));
// \Illuminate\Support\Facades\Log::channel('custom_channel')->info('$body ' . PHP_EOL . print_r($body, true));
return $response;
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function createMeeting(array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings';
return $this->makeRequest($endpoint, 'POST', $payload);
}
/**
* @throws BadRequest
* @throws HubspotException
*/
public function updateMeeting(string $meetingId, array $payload): Response
{
$endpoint = '/crm/v3/objects/meetings/' . $meetingId;
return $this->makeRequest($endpoint, 'PATCH', $payload);
}
/**
* @throws \Exception
*/
public function createNote(
string $body,
string $ownerId,
int $timestamp,
string $objectId,
NoteObject $noteObject
): ?string {
try {
$noteInput = new SimplePublicObjectInput([
'properties' => [
'hs_note_body' => $body,
'hubspot_owner_id' => $ownerId,
'hs_timestamp' => $timestamp,
],
]);
// Create note
$note = $this->getNewInstance()->crm()->objects()->basicApi()->create('note', $noteInput);
$this->getNewInstance()->crm()->objects()->associationsApi()->create(
'note',
$note->getId(),
$this->getNoteObject($noteObject),
$objectId,
$this->getNoteAssociationType($noteObject),
);
return $note->getId();
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to create note', [
'objectId' => $objectId,
'noteObject' => $noteObject->getObjectType(),
'reason' => $e->getMessage(),
]);
\Sentry::captureException($e);
}
return null;
}
public function updateEngagement(string $objectId, array $engagement, array $metadata): void
{
$this->getInstance()->engagements()->update($objectId, $engagement, $metadata);
}
public function getEngagementData(string $engagementId): array
{
$engagement = $this->getInstance()->engagements()->get($engagementId);
return $engagement->toArray();
}
public function createEngagement(array $engagement, array $associations, array $metadata): Response
{
return $this->getInstance()
->engagements()
->create($engagement, $associations, $metadata);
}
public function isUnauthorizedException(\Exception $e): bool
{
// Check for specific HubSpot API exception types first
if ($e instanceof BadRequest) {
// BadRequest can contain 401 status codes
return $e->getCode() === 401;
}
// Check for HTTP client exceptions with status codes
if ($e instanceof \GuzzleHttp\Exception\RequestException && $e->hasResponse()) {
$response = $e->getResponse();
if ($response !== null) {
return $response->getStatusCode() === 401;
}
}
// Check for Guzzle HTTP exceptions
if ($e instanceof \GuzzleHttp\Exception\ClientException) {
return $e->getCode() === 401;
}
// Fallback to string matching as last resort, but be more specific
$message = strtolower($e->getMessage());
return str_contains($message, '401 unauthorized') ||
str_contains($message, 'http 401') ||
str_contains($message, 'status code 401') ||
(preg_match('/\b401\b/', $message) && str_contains($message, 'unauthorized'));
}
/**
* Validates and refreshes the access token if needed before API requests.
* This ensures long-running processes don't fail due to token expiration.
*
* @throws SocialAccountTokenInvalidException
*/
public function ensureValidToken(): void
{
if ($this->oauthAccount === null) {
return;
}
$newToken = $this->tokenManager->ensureValidToken($this->oauthAccount);
if ($newToken !== null) {
$this->accessToken = $newToken;
}
}
public function getConfig()
{
return $this->config;
}
// returns only active (archived=false)
public function getOwners(): array
{
return $this->getNewInstance()->crm()->owners()->getAll();
}
/**
* @param bool $archived
*
* @return array<Owner>|[]
*/
public function getOwnersArchived(bool $archived = true): array
{
$endpoint = '/crm/v3/owners';
$queryParams = [
'archived' => $archived ? 'true' : 'false',
];
$queryString = http_build_query($queryParams);
$owners = [];
try {
$response = $this->makeRequest(endpoint: $endpoint, queryString: $queryString);
$responseData = $response?->toArray();
foreach ($responseData['results'] as $result) {
try {
$owners[] = Owner::create($result);
} catch (Throwable $e) {
$this->log->error('[HubSpot] Failed to process owner data', [
'result' => $result,
'error' => $e->getMessage(),
]);
continue;
}
}
} catch (Throwable $e) {
$this->log->error('HubSpot] Failed to fetch owners', [
'archived' => $archived,
'error' => $e->getMessage(),
]);
return [];
}
return $owners;
}
public function getMeeting(string $engagementId): ObjectWithAssociations
{
return $this->getNewInstance()->crm()->objects()->basicApi()
->getById('meeting', $engagementId, null, 'contact,company,deal');
}
public function deleteEngagement(string $engagementId): void
{
$this->getInstance()->engagements()->delete((int) $engagementId);
}
public function getAssociationsData(array $ids, string $fromObject, string $toObject): array
{
$associationData = [];
$idChunks = array_chunk($ids, self::ASSOCIATIONS_BATCH_SIZE_LIMIT);
foreach ($idChunks as $idChunk) {
try {
$batchInput = new \HubSpot\Client\Crm\Associations\Model\BatchInputPublicObjectId();
$batchInput->setInputs(array_map(function ($id) {
$publicObjectId = new \HubSpot\Client\Crm\Associations\Model\PublicObjectId();
$publicObjectId->setId($id);
return $publicObjectId;
}, $idChunk));
$associatedObjectsData = $this
->getNewInstance()
->crm()
->associations()
->batchApi()
->read($fromObject, $toObject, $batchInput);
if ($associatedObjectsData instanceof \HubSpot\Client\Crm\Associations\Model\BatchResponsePublicAssociationMulti) {
foreach ($associatedObjectsData->getResults() as $association) {
$from = $association->getFrom()->getId();
$toAssociations = $association->getTo();
if (! empty($toAssociations)) {
$associationData[$from] = array_map(function ($item) {
return $item->getId();
}, $toAssociations);
}
}
}
// } catch (RateLimitException $e) {
// throw $e;
} catch (\Exception $e) {
$this->log->error('[Hubspot] Failed to fetch associations', [
'from_object' => $fromObject,
'to_object' => $toObject,
'reason' => $e->getMessage(),
]);
}
}
return $associationData;
}
/**
* @throws \Exception
*/
private function getNoteAssociationType(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'note_to_deal',
NoteObject::Lead, NoteObject::Contact => 'note_to_contact', // or 'note_to_lead' if your portal supports it
NoteObject::Account => 'note_to_company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
/**
* @throws \Exception
*/
private function getNoteObject(NoteObject $noteObject): string
{
return match($noteObject) {
NoteObject::Opportunity => 'deal',
NoteObject::Lead, NoteObject::Contact => 'contact',
NoteObject::Account => 'company',
NoteObject::Call, NoteObject::Event => throw new \Exception('Not supported'),
};
}
public function addAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/create";
return $this->makeRequest($endpoint, 'POST', $payload);
}
public function removeAssociations(string $objectType, string $associationType, array $payload): Response
{
$endpoint = "/crm/v4/associations/$objectType/$associationType/batch/archive";
return $this->makeRequest($endpoint, 'POST', $payload);
}
}
Sync Changes
Hide This Notification
Code changed:...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15642
|
|
15641
|
PhpStormVIewINavicareCodeLaravelKeractorFV faVsco. PhpStormVIewINavicareCodeLaravelKeractorFV faVsco.js?9 JY-20725-handle-HS-search-rate-limitProletey© HubspotPaginationService.phpC HubSpot/Service.pnp© SyncRelatedActivityManager.php=custom.log~=laravel.logA SF (jiminny@localhost]4 HS_local [jiminny@localhost]© BatchSyncCollectol© HubspotSyncStrategyBase.phpy syneermenttes tralt.onpCachedcrmservicebecorator.onp© ProspectCache.php# console [PKob.# console leu)# console [slAGiNg)e balchsynckealsseС Cпескапокetrукemotematch.ong[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {W19лVc clientoneccloseaDealstagess @ MatchacuivitycrmData.ong© CrmActivityService.phg"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"JnDealrielasservice.g© CrmObiectsResolver.php"concenc-lyoe. apolicacionison.charser=utt-onc)Decorateacuivilv.or© FieldDefinitions.phr"Transter-Encod1nq":"chunked").class Elzent extends Baseulzent implements hubspotelzentintertaceм A2 A65 ×1 M1 лC) FieldT vpeconverte"Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],e Hubspotclientinterpublic function ensureValidTokenO: voidc) Hubspotlokenman"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncLudeSubDomains: preload")nC) PavloadBuilder.phpif (Sthis->oauthAccount === nulb) {C) RemotecrmobiectrP ResponseNormalizeaccent-encodino"c) Service,onoC)SvncFieldAction.on"access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\",C) SvncRelatedActivitC) WebhookSvncBatclSnewlloken=sthis->tokenmanagen->ensureva.1dlokencsth1s->oauthAccount)if (SnewToken |== null) iSthis-›accessToken = $newToken;v MintearationAor"x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"Sot-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtmoV-1778163675-1010.1M Acceccors88907-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],• D ConfigD DTO> D FiltersD Jobs• M ProcnectSoarchStr.public function getConfigo"Report-To":["{"endnoints"."unl".httns:la.nel.cloudflane.com./renon+W/v42c=NVA1cVTPQfVm32anS0#xVF/sd2Rlreturn $this->config;W service lralts© DataClient.php/I returns only active (archived=false)"group\":\"cf-nel\","max_age\":604800}"],"NEL":["{success_traccion.0.01r9 usagesreportto. "cr-nel,Console,Log XChanaes 12 filed= env.local aonTJ0 + → Side-by-side viewer •Do not ianorey8 35f036ac app/Services/Crm/Hubspot/Client.phgSthis->setVersion(self::MIN_API_VERSION):Hiahliaht wordsyXBB ?© HandleHubspotRateLimit.php app/Jobs/Middleware© HubspotClientinterface.php app/Services/Crm/Hubspot© HubspotPaginationService.php app/Services/Crm/Hubspot/Pagination@ JiminnyDebugCommand.php app/Console/Commandsphp logging.php config© MatchActivityCrmData.php app/Jobs/Crm© MatchCrmData.php app/Jobs/Activity/Import© PaginationState.php app/Services/Crm/Hubspot/Pagination© RateLimitException.php app/Exceptions© Service.php app/Services/Crm/HubspotUnversioned Files 9 files, updating..ououic tunction cetranmumaoiversiono: strinoreturn self:: MIN APT VERSTON:SlastRecordtdi= env.nikilocal apoE.env.other app©) CanAccessAiReportsTest.php tests/Unit/Policies• CreateMockAskJiminnvReportResultCommand.php app/Console/Comn* Athrows NealAniFycention* Athrows ErmSycentionki tavicon.ico publicE ids txt aprpublic function getOpportunityById(string $crmId, array $fields): arrayiaraw sol querv sall aool© SimulateWebhooksCommand.php app/Console/Commands/Crm/Hubspottry{Sdeal = Sthis->getNewInstance->crm->deals->basicApi->qetById(olner Code will hoin INF to underctand vour Laravel ann code II Generate II Don't Show Anvmore (58 minutes aao)40 hll"supoont Dally • In 4n 04m100% C47 • Mon 11 May 10:06:14AskJiminnyReportActivityServiceTest v+0 ..CascadeHubspot Rate Limit ReClient.php:516-523}catch (ContactApiException $e) ‹Sthis->log→>info('[Hubspot] Failed to fetch contact', ['email" => Semail,'reason' => $e->getMessage(),return []:It returns [). From the job's point of view, that's identical to "this email is not in HubSpot". So1. matchExactlyByEmail silently swallows the 429.2. The cascade falls through to matchByPhone (another HubSpot API call, this time the search endpoint).3. We just made an additional API call while already rate-limited — amplifvina the pressure.4. If matchByPhone returns nothing too, the activity gets persisted as "no CRM match" even though the contact exists in HubSpot. Wrong data, no signal to retry.The middleware on MatchAct ivitvCrmData cannot helo, because the 429 was already swallowed before anv RateL imitExcention could be raised.How serious is it reallv?In practice — probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the APl add-on). which is much more aenerous than the search endooint (5 RPS). 429s on this endooint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail.Ask anything (&AL)+ <> CodeClaude Opus 4.7 MediumQ differencesCurront vorcionreturn 10:public function getMinimumApiVersion@: strinareturn self::MIN APT VERSION:SlastRecordId* Execute a search request aanins+ HubSnot CRM obiects with rate limitinal* Qparam string $objectType The object type ('deals','companies', 'contacts', 'calls')* @panam array<string, mixed> $payload The search payload with filters, sorts, properties, etc.* @return array The search response with 'results', 'total', 'paging' keys* Athrows RateLimitException When rate limit is hit+ Athnowe Hubeno+Gycention An APT ennonclWN Windsurf Toams 885•62UTF.8Po 4 spaces...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15641
|
|
15640
|
Project: faVsco.js, menu
JY-20725-handle-HS-search Project: faVsco.js, menu
JY-20725-handle-HS-search-rate-limit, menu
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жзSupport Daily - in 4h 54 mDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:06:14181screenpipe"0 ₴6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15640
|
|
15639
|
Firefox FileEditViewHistoryBookmarksProfilesToolsW Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жзSupport Daily - in 4h 54 mDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:06:11181screenpipe"0 ₴6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15639
|
|
15638
|
SlackmistoryWindowrTavsco.s?9 JY-20725-handle-HS-s SlackmistoryWindowrTavsco.s?9 JY-20725-handle-HS-search-rate-limitroledey© HubspotPaginationService.php© SyncRelatedActivityManager.php© BatchSyncCollector© HubspotSyncStrategyBase.phpCachedcrmservicebecorator.onp© ProspectCache.phpe balchsynckealsseС Cпескапокetrукemotematch.ongc clientonec closeaDealstagess @ MatchacuivitycrmData.ong© ermactivilyservice.phgDealrielasservice.gc)Decorateacuivilv.or© FieldDefinitions.phrclass Cllent extends Baseclient 1mpLements Hubspotclientintertace- A2 A65 X1X1 ~C) FieldT vpeconvertee Hubspotclientinterc) Hubspotlokenman© PayloadBuilder.phpC) RemotecrmobiectnP ResponseNormalizec) Service,onoC)SvncFieldAction.onC) SvncRelatedActivitC) WebhookSvncBatclv MintearationAorM AcceccorsConfigD DTO• M SiltersJobs• M ProcnectSoarchStr.W service lralts© DataClient.php© DecorateActivity.phcLocalsearch.oneu LocalSearchintertac© RemoteSearch.phpc) Service.phpv W Listenersc) ConvertLeadActivitc) PurceLookuocache> M Metadata> Miarationa Pioedrivev Salesforce• D Fields• M OnnortunitvMatcheMOnnortunitvSvneSt897 (> M ProsnectSearchStr:M ServiceTraitcC) Client nhr© DecorateActivity.ph. Delete@biectsTrait© FieldDefinitions.php© PayloadBuilder.php© Profile.php© QueryBuilder.phpoubuc tunction 1sunauthorizedzxcention Excention se: 000ureturn str contains(Smessage. "401 unauthorized')Istr contains(Smessage.'http 401') |1str_contains(Smessage, 'status code 401') |(orea match( pattern: • /\b401\b/* Smessage) &s str contains(Smessage. 'unauthorized')):* Validates and refreshes the access token if needed before API requests.* This ensures long-running processes don't fail due to token expiration.* @throws SocialAccountTokenInvalidExceptiony usagespublic function ensureValidToken: voidif (Sthis->oauthAccount === null) {Snewloken = sthis->tokenmanaqer->ensureval1dloken(sth1s->oauthaccount)rif (SnewToken !== null) {sthis-›accessloken = snewloken:public function getConfigoreturn Sthis->confia:returns onlv active (anchived=false)9 usadespublic function get0wners: arrayreturn $this->getNewInstance()->crm()->owners()->getALL();* apanam bool Sarchived* dreturn arrau<ownens/rn1s usagesS0 hill• suppont Dally • In 41 04m100% L2• Mon 11 May 10:06:08+0 ..11 1111.1III=custom.log~=laravel.logA SF (jiminny@localhost]4 HS_local (jiminny@localhost]# console [PKob.# console leu)# console [slAGiNg)[2026-05-07 14:21:15] local.INF0: [Hubspot] DEBUG Getting headers {"neaders".?"Uace":L"Inu,or May 2020 14.21.15 6Ml"Jn"Transter-Encod1nq":"chunked")."Connection":"keep-alive""CF-Ray" : ["9f80deb8db60dc3a-SOF"],"Strict-Transport-Secur1ty":"max-aqe=31536008* 1ncludeSubDomains: preload")naccent-encodino""access-control-allow-credentials": ["false"]."server-timing": ["hcid;desc=\"019e02d0-6fd8-7812-bdba-885b7ccb3ee3\","x-hubspot-correlation-id":["019e02d0-6fd8-7812-bdba-885b7ccb3ee3"],"So+-Cookie"." c+hm-Stlirtd0aXVr.kSandas6hzVVKhzTn0BidvMaheCtm0V-1778163675-1.0.107-May-26 14:51:15 GMT; domain=.hubapi.com; Http0nly; Secure; SameSite=None"],"Renont-To"."","endpoints\":[{"unl".httns:a.nel.cloudflane.com./nenon+|W/v42c=NVA1cVTPQfVm32anS0axVF/sd2R"group\":\"cf-nel\","max_age\":604800}"],"NEL":["{\"success_fraction\":0.01,reportto. "cr-nel,"max age":604800}"]"Server": ["cloudflare"]}} {"correlation_1d":"95256555-ec98-4541-b9za-adta/Sb6Yeab","trace_10":"C/AD8565-905t-4604-9405-0e50551e5545I IIllC;CascadeHubspot Rate Limit ReClient.php:516-523}catch (ContactApiException $e) ‹Sthis->log→>info('(Hubspot] Failed to fetch contact', ['email" = Semail.'reason' => $e->getMessage(),return []:It returns [). From the job's point of view, that's identical to "this email is not in HubSpot". So1. matchExactlyByEmail silently swallows the 429.2. The cascade falls through to matchByPhone (another HubSpot API call, this time the search endpoint).3. We just made an additional API call while already rate-limited — amplifvina the pressure.4. If matchByPhone returns nothing too, the activity gets persisted as "no CRM match" even though the contact exists in HubSpot. Wrong data, no signal to retryThe middleware on MatchAct ivitvCrmdata cannot helo, because the 429 was already swallowed before anv RateLimitExcention could be raised.How serious is it realiv?In practice - probably rare, but still wrong. The GET-by-email endpoint shares HubSpot's portal-wide burst quota (100/10s on Free/Starter, 150/10s on Pro+200/10s with the APl add-on). which is much more aenerous than the search endooint (5 RPS). 429s on this endooint are uncommon. But:• During heavy backfills they do happen — the same worker pool that's hammering search will also be hammering getContactByEmail.• When it does happen, the cost is silent data corruption (wrong CRM match) plus extra wasted API calls falling through the cascadeWhat vou should decideGiven your simplification (only Client::search() raises RateLimitException, only MatchActivityCrmData middleware handles it), you have two reasonableoptions:Ontion A- Accent the aan documentltThe pris sconelis "handle search endnoint ratellmits" since fhatis where 429s actualvhurt EmaiEbvard 429s are rareand currently treated as "not found" - same as before this PR. Nothing breaks, you just don't fix this particular gap. Add a TODO/follow-up ticket.Option B - Wrap getContactByEmail in executeRequest() too. Five-line change, then the email-match step would also throw RateLimitException → middlewarereleases the lob. his s cheao.and.removes the silent-corruption.risk for that one.method.You don't need to touch netcontactByd/netOonortunityßy dletchecauce thoce aren't on the MatchActivitvGrmlata hot nathlMy recommendation: Option B. It's a tiny addition that closes the matching cascade's first step, which is the most-hit path. Without it, the new middleware missesthe most common entry point of MatchActivityCrmData's CRM lookups• phppublic function getContactBynail(string Senail, array stields = Il): arraytrydtcontort - Cthie,rovocntobonnoctfn () = Sthis->getNewInstance()->crm()->contacts()->basicApi()->getById($email, implode(',', $fields), null, false,"emai1")return ['id' => $contact->getId(), 'properties' => $contact->getProperties()]:} catch (RateLimitEycention se)f} catch (ContactApiException Se) {Ask anvthina (&4L)+ « CodeClaude Onus 4.7 Medium802-6Pa 4 sp...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15638
|
|
15637
|
FirefoxFileEditViewHistoryBookmarksProfilesToolsWi FirefoxFileEditViewHistoryBookmarksProfilesToolsWindowHelp# Support Daily - in 4 h 54 mDEV (docker)• жзDOCKER881DEV (docker)H82APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY].JY-JY-M-webPS$IggeDI-ionJY-20326factorinfo-messagecommandJY-20317-fix-auto-log-delay-issue-on-aJ}channels-disabledJY-20312-remove-on-update-change-last-synced-vnced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-if-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminnyroot@docker_lamp_1:/home/jiminny#~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ dev-zsh-zsh885100%8• Mon 11 May 10:06:08L₴81screenpipe"0 ₴6DEV...
|
PhpStorm
|
faVsco.js – Client.php
|
NULL
|
15637
|
|
15636
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Toast
Jira Cloud
Google Calendar
Messages
Messages
Add canvas
Add canvas
Files
Files
Pins
Pins
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
May 8th at 2:53:03 PM
2:53
опитвам се да оптимизирам процеса по индексиране на активитита за ЕС
May 8th at 2:54:16 PM
2:54
идеята е да намаля паметта която се ползва за да се генерира един бач от 100 активитита
May 8th at 2:54:35 PM
2:54
и после да увелича размера на бачовете, за да имаме по малко blocking операции в ЕС, като реиндексира
Lukas Kovalik
May 8th at 4:12:58 PM
4:12 PM
здрасти, изглежда ок, но когато го минах и през gemini ми даде един warning.
May 8th at 4:13:17 PM
4:13
Switch
cursor()
to
lazyById(250)
. It preserves the single-loop, generator-style code in the new version while restoring proper batched eager loading (avoiding N+1 on
getIndexableAttributes()
) and releasing the DB connection between chunks (avoiding long-held PDO connections during ES/Sentry calls).
Vasil Vasilev
May 8th at 4:40:06 PM
4:40 PM
хм, интересна идея
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:41:03 PM
4:41
ще го проверя
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:41:17 PM
4:41
така или иначе в понеделник ще иде на прод, днес не ми се рискува
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Vasil Vasilev
May 8th at 6:05:10 PM
6:05 PM
мм, прав си
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:05:26 PM
6:05
задълбах по на дълбоко, и се оказва, че cursor ще произведе друг проблем
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:05:35 PM
6:05
cursor не зарежда допълннителните таблици
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:07 PM
6:06
ще стриймне моделите, но за всеки ще вика поотделно with(opportunities, leads, accounts….) и всичко останало
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:16 PM
6:06
което обезмисля цялата гимнастика
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:21 PM
6:06
ще го ревизирам с lazy loading
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:43 PM
6:06
на по малки бачове, за да спестя памет от моделите, просто ще взема данните на няколко пъти
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
Channel
slackcalVIewJiminny …..vHomeActivityFllesLaterMore# curiosity_lab# engineering# general# jiminny-bg# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of jimi...• Direct messages€. Vasil VasilevC. Nikolay IvanovP. Galya Dimitrova3 Aneliya Angelova, ...8o Stoyan Tanev8 Stefka Stovanova@ VesA. Aneliya AngelovaL James Graham. Lukas Kovalik y...i Apps® ToastG Jira CloudMistonWindowhelp@ Describe what you are looking forC. Vasil VasilevMessagest Add canvasUr Files& Pinsproper D: Friday. May 8th v 3 (avoiding N+1on getIndexableAttributes()) andreleasing the Do connecnon detweenchunks (avoiding long-held PDOconnections during ES/Sentry calls.Vasil Vasilev 4:40 PMхм. интересна идеяше го проверятака или иначе в понеделник ще иде напрод, днес не ми се рискуваVasil Vasilev 6:05 PMмм, прав сизадълбах по на дълбоко, и се оказва, чеcursor ше произведе друг проолемcursor не зарежда допълннителнитетаблицище стриймне моделите, но за всеки щевика поотделно with(opportunities, leads.ecelliaitnneukeoletnlolкоето обезмисля пялата гимнастикаще го ревизирам с lazy loadingна по малки бачове, за ла спестя паметот моделите, просто ще взема даннитена няколко пьтиMessage Vasil Vasilev+ Aa IVasil-Jiminny and others added 28 commits 4 days agoQ Type D to searchly and qualityw InsightsLộs Settingsu opt out. Review this update and manage vour preterences in vour GitHub account settings.ydration #12059 °• Checks pendingCode -Y-20820-es-reindex-stream-model-hydrations changed 12+262 .140 6882lReviewers@ yalokin-jiminny€ vkulovLakyLakngle model, that is hydrated, extracts indexing data,aded, with a massive overhead lust tor keenina aStill in progress? Convert to draftAssigneesnail filters were added to I oadDocuments. so thatit" command to ES for that document. This leads toNo one—assign yoursellorocesses, start command 'runForSeconds' is notNone yetProiectsNone vetMilestoneNo milestoneDevelopmentSuccessfully merging this pull request may close theseissues.None vetAdd "isDeleted" accessor methods for usage in building ES payload.adc64baRemove method "isDeleted" from ActivityElasticSearchTrait as it's alr_ ...3b39a46NotificationsUnsubscribe• Add isDeleted to the Searchable contract.3ca67a06- Move the logic of Skio Emails into the elastic search activity reposi. ...c7b2fb5€ Since skipping emails is now delegated to the DB, we can remove artif..470e119nartcnontea7eh4?4FavouritesE jiminny(* AirDrop• RecentsA ApplicationsDocuments© Downloadsii lukasiCloud• iCloud Drive283 Sync folderLocationsC DXP4800PLUS-B5F 4@ Network• CRM• Orange• Red• Yellow• Green• Blue• PurpleO All Tags...workNameV 2026= Daily 2026-05-11.mp4Daily 2026-05-08.mp411-1 2020-00-0/.mp4I Dailv 2026-05-07mo4* 1-1 2026-04-24.mp4= Daily 2026-04-24.mp4r User Pilot introduction Adi 2026-04-23.mo4m Daily 2026-04-23.mp4Daily 2026-04-22.mp4n Refinement 2026-04-06.mp4- Daily 2026-04-21.mn4Du Refirnent 2026-04-20.mp4Dally 4020-04-40.mp4* Dailv 2026-04-17 mo4Ta Daily 2026-04-16.mp4a Planning 2026-04-15.mp4Retro 2026-04-14.mp4Daily 2026-04-14 mn4|User pilot (Adi) 2026-04-09.mp4—Dally 2020-04-09.m04w Dailv 2026-04-08.m04• Daily 2026-04-07.mp4Daily 2026-04-06.mp4= Dailv 2026-04-03.mo4es Planning 2026-04-01 & task split.mp4m Dally 2026-03-31.mp4•: Refinement 2026-03-30.mo4m Daily 2026-03-30.mp4• Daily 2026-03-27.mp4= Dailv 2026-03-26.mo4|- Daily 2026-03-24.mp4Refinement 2026-03-23.mp4= Daily 2026-03-23.mp4•a RE chanter 2026-02-20 mn/Daily 2026-03-20.mp4am Planing 2026-03-18-converted.mp4#. Refinement 2026-02-09-converted.mp4RR Daily 2026-03-19.mp4- Review 2026-03-18.mp4# Planina 2026-03-18.mp4- Petra 2026-02-17 mn/- Daily 2026-03-17.mp4- Refinement 2026-03-16.mp4• Dailv 2026-03-16.mn/1m Daily 2026-03-13.mp4Dailv 2026-03-12.mp4A Daily 2026.02-11 mл/• Daily 2026-03-10.mp4*: Refinement 2026-03-09.mp4ea Daily 2026-02-09 mn/fm Daily 2026-03-06.mp4a Planning 2026-03-04.mp4- Dailv 2026-03-02 mo4$0Support Daily - in 4h 54m100% 2Mon 11 May 10:06:06vQ Sear!Date ModifiedToday at 10:04Today at 10:028 May 2026 at 10:22/May 2026 at 18:217 Mav 2026 at 10:1024 Apr 2026 at 14:4424 Apr 2026 at 10:1123 Aor 2026 at 11:5823 Apr 2026 at 10:3221 Aor 2026 ar 11:0221 Aor 2026 at 10:0020 Apr 2026 at 16:5620 Apr 2026 at 10:0617 Aor 2026 at 10:1616 Apr 2026 at 10:0015 Apr 2026 at 11:1414 Aor 2026 at 17:3714 Anr 2026 at 10:009 Apr 2026 at 14:479 Apr 2026 at 10:078 Aor 2026 at 10:137 Apr 2026 at 10:016 Apr 2026 at 10:083 Aor 2026 at 10:211Anr 2026 at 12:2031 Mar 2026 at 18:2931 Mar 2026 at 10:1030 Mar 2026 at 17:1230 Mar 2026 at 10:0527 Mar 2026 at 10:0926 Mar 2026 at 9:59Mer 0023 Mar 2026 at 10:0020 Mar 2026 at 11:1620 Mar 2026 at 10:0619 Mar 2026 at 12:0119 Mar 2026 at 11:3519 Mar 2026 at 9:5718 Mar 2026 at 16:2018 Mar 2026 at 11:1417 Mar 2026 at 17:1017 Mar 2026 at 10:1816 Mar 2026 at 16:5516 Mar 2026 at 10:0713 Mar 2026 at 10:1212 Mar 2026 at 10:1011 Mar 2026 at 10:0610 Mar 2026 at 9:579 Mar 2026 at 17:04Q Mar 2026 at 0:566 Mar 2026 at 9:572 Mar 2026 at 10:07Folder101.2 MR1,37 CBMPEG-4 movie1,55 GBMPEG-4 movie931.7 MEMPEG-4 movie1,86 GB MPEG-4 movie832,2 MBMPEG-4 movie724 M:MPEG-4 movie1,74 GBMDSG-A movie1,36 GB2,41 GBMP2G-4 movie567 8 MEMPEG-4 movie4,25 GBMPEG-4 movie698,5 MBMPEG-4 movie1.16 GEMPEG-4 movie513,4 MB MPEG-4 movie2,75 GB1.44 G:MPEG-4 movie9244MRMPEG-A movid362,6 MBMPEG-4 movie748,8 MBMPEG-4 movie1.04 GEMPEG-4 movie575,5 MB MPEG-4 movie1.02 G:MPEG-4 movieMRQCpMDEG.A movic3,4 GBMPEG-4 movie923,6 MBMPEG-4 movie2.77 GEMPEG-4 movie641,8 MBMPEG-4 movie884,3 MB476.6 M:MPEG-4 movie550 9 MEMDEC A movid3,44 GBMPEG-4 movie438,9 MBMPEG-4 movie169 GPMDEC-A movie430,4 MB MPEG-4 movie2,38 GB2.26 G:MPEG-4 movie386,3 MBMDEC.A movid2.78 GBMPEG-4 movie152 GPMDEG-A movid1,2 GBMPEG-4 movie4,19 GB592 2 M:MPEG-A movid1,02 GBMPEG-4 movie978.7 MBMPEG-4 movie709 7 MPMDEG.A movid404,6 MB MPEG-4 movie4,16 GB219 7 MPMPEG-A movid291.7 MBMPEG-4 movie768,5 MB MPEG-4 movie1 of 153 selected. 1.94 TB available...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15636
|
|
15635
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Toast
Jira Cloud
Google Calendar
Messages
Messages
Add canvas
Add canvas
Files
Files
Pins
Pins
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
May 8th at 2:53:03 PM
2:53
опитвам се да оптимизирам процеса по индексиране на активитита за ЕС
May 8th at 2:54:16 PM
2:54
идеята е да намаля паметта която се ползва за да се генерира един бач от 100 активитита
May 8th at 2:54:35 PM
2:54
и после да увелича размера на бачовете, за да имаме по малко blocking операции в ЕС, като реиндексира
Lukas Kovalik
May 8th at 4:12:58 PM
4:12 PM
здрасти, изглежда ок, но когато го минах и през gemini ми даде един warning.
May 8th at 4:13:17 PM
4:13
Switch
cursor()
to
lazyById(250)
. It preserves the single-loop, generator-style code in the new version while restoring proper batched eager loading (avoiding N+1 on
getIndexableAttributes()
) and releasing the DB connection between chunks (avoiding long-held PDO connections during ES/Sentry calls).
Vasil Vasilev
May 8th at 4:40:06 PM
4:40 PM
хм, интересна идея
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:41:03 PM
4:41
ще го проверя
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:41:17 PM
4:41
така или иначе в понеделник ще иде на прод, днес не ми се рискува
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Vasil Vasilev
May 8th at 6:05:10 PM
6:05 PM
мм, прав си
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:05:26 PM
6:05
задълбах по на дълбоко, и се оказва, че cursor ще произведе друг проблем
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:05:35 PM
6:05
cursor не зарежда допълннителните таблици
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:07 PM
6:06
ще стриймне моделите, но за всеки ще вика поотделно with(opportunities, leads, accounts….) и всичко останало
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:16 PM
6:06
което обезмисля цялата гимнастика
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:21 PM
6:06
ще го ревизирам с lazy loading
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 6:06:43 PM
6:06
на по малки бачове, за да спестя памет от моделите, просто ще взема данните на няколко пъти
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
loading…
Channel
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жзSupport Daily - in 4h 54 mDOCKER881DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY]@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:06:041881screenpipe"O 86DEV...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15635
|
|
15634
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Toast
Jira Cloud
Google Calendar
Messages
Messages
Add canvas
Add canvas
Files
Files
Pins
Pins
Add and Edit Channel Tabs
Canvas
List
Folder
Jump to date
Lukas Kovalik
Apr 28th at 4:53:41 PM
4:53 PM
ок, ще питам Вес, мерси
Vasil Vasilev
Apr 28th at 5:00:16 PM
5:00 PM
моля
Jump to date
Vasil Vasilev
May 8th at 2:52:43 PM
2:52 PM
Лукаш, привет
May 8th at 2:52:48 PM
2:52
хвърли моля те едно око тука
May 8th at 2:52:49 PM
2:52
https://github.com/jiminny/app/pull/12059
https://github.com/jiminny/app/pull/12059
May 8th at 2:53:03 PM
2:53
опитвам се да оптимизирам процеса по индексиране на активитита за ЕС
May 8th at 2:54:16 PM
2:54
идеята е да намаля паметта която се ползва за да се генерира един бач от 100 активитита
May 8th at 2:54:35 PM
2:54
и после да увелича размера на бачовете, за да имаме по малко blocking операции в ЕС, като реиндексира
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
slackcalVIewJiminny …..vHomeActivityFllesLaterMore# curiosity_lab# engineering# general# jiminny-bg# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of jimi...• Direct messages€. Vasil VasilevC. Nikolay IvanovP. Galya Dimitrova3 Aneliya Angelova, ...8o Stoyan Tanev8 Stefka Stovanova@ VesA. Aneliya AngelovaL James Graham. Lukas Kovalik y...i Apps® ToastG Jira CloudMistonWindowhelp@ Describe what you are looking forC. Vasil VasilevMessagest Add canvasur FilesX Pinsи после да увелича размера на оачовете.за да имаме по малко blocking операциив сс, като реиндексираLukas Kovalik 4:12PMздрасти, изглежда ок, но когато го минахи през детіnі ми даде един warning.Switch cursor( to LazyById(250)]. Itpreserves the single-loop, generator-stylecode in the new version whlle restoringproper batched eager loading (avoiding N+1on qetindexableAttributeso anareleasing the DB connection betweenchunks avoiding long-nelc rueconnections during ES/Sentry calls).Vasil Vasilev 4:40 PMхм. интересна идеяще го проверятака или иначе в понеделник ще иде напрод, днес не ми се рискуваVasil Vasilev 6:05 PMмm nnap culзадълбах по на дълбоко, и се оказва, чеcursor ще произведе друг проблемcursor не зарежда допълннителнитеMessage vasil vasilev+ Aa IVasil-Jiminny and others added 28 commits 4 days agoQ Type D to searchty and qualltyw InsightsLộs Settingsu opt out. Review this update and manage vour preterences in vour GitHub account settings.ydration #12059 •Checks pendingCode -1-20820-es-reindex-stream-model-hydrations changed 12+262 .140 6882lyalokin-jiminny€ vkulovLakyLakngle model, that is hydrated, extracts indexing data,aded, with a massive overhead lust tor keenina aStill in progress? Convert to draftAssigneeslail filters were added to l oadDocuments. so thatNo one—assign yoursellIt" command to ES for that document. This leads toprocesses, start command 'runForSeconds' is notNone yetProiectsNone vetMilestoneNo milestoneDevelopmentSuccessfully merging this pull request may close theseissues.None vetAdd "isDeleted" accessor methods for usage in building ES payload.adc64baRemove method "isDeleted" from ActivityElasticSearchTrait as it's alr_ ...3b39a46NotificationsUnsubscribeAdd isDeleted to the Searchable contract.3ca67a06) Move the logic of Skio Emails into the elastic search activity reposi.. ..c7b2fb5€ Since skipping emails is now delegated to the DB, we can remove artif..470e119nartcnontea7ehd?tFavouritesE jiminny(* AirDrop• RecentsA ApplicationsDocumentsii lukasiCloud• iCloud Drive283 Sync folderLocationsC DXP4800PLUS-B5F A® Network• CRM• Orange•Red• Yellow• Green• Blue• PurpleO All Tags...workNamev N 2026- Daily 2026-05-11.mp4* Dally 2020-00-08.mp4к 1-1 2026-05-07mo4#a Daily 2026-05-07.mp4* 1-1 2026-04-24.mp4= Daily 2026-04-24.mp4m User Pilot introduction Adi 2026-04-23.mp4Daily 2026-04-23.mp4Daily 2026-04-22.mp4*m Refinement 2026-04-06.mp4Daily 2026-04-21.mp4Du Refinement 2026-04-20.mp4zz Daily 2026-04-17.mp4ww Planning 2026-04-15.mp4Retro 2026-04-14.mn/• Daily 2026-04-14.mp4User pilot (Adi) 2026-04-09.mp4• Dailv 2026-04-09,mo4w: Daily 2026-04-08.mp4• Daily 2026-04-07.mp4• Dallv 2026-04-06.mo4= Daily 2026-04-02 mn4ining 2026-04-01 & task split.mp4i: Retro 2026-03-31.mp4em Dailv 2026-03-31.mo4- Refinement 2026-03-30.mp4- Dallv 2026-03-27mo4• Daily 2026-02-26 mn4- Daily 2026-03-24.mp4= Refinement 2026-03-23.mp4• Dailv 2026-03-23.mo4•* BE chapter 2026-03-20.mp4- Daily 2026-03-20.mp4a Planina 2026-03-18-converted.mo4|- Refinement 2026-02-09-converted.mp4• Review 2026-03-18.mp4aa Planina 2026-03-18.mo4v Retro 2026-03-17.mp4- Daily 2026-03-17.mp4• Refinement 2026-03-16.mo4|- Daily 2026-03-16.mp4Daily 2026-03-13.mp4* 1-12026-03-12.mp4Daily 2026-03-12 mn/ea Daily 2026-03-11.mp4Daily 2026-03-10.mp4xE: Refinerant 2026-03-09 mo4вaлail bane лe,л0 mnлl* Plannina 2026-03-04.mo4C nail 2026.02.02 mm/l$0Support Daily - in 4h 55m100% 28• Mon 11 May 10:05:35Q SeaiDate ModifiedTodav at 10:048 May 2026 at 10:227 May 2026 at 18:217 May 2026 at 10:1024 Apr 2026 at 14:4424 Apr 2026 at 10:1122 Anr 2026 at 11:5923 Apr 2026 at 10:3222 Apr 2026 at 10:2171 Aor 2026 at 11:0721 Apr 2026 at 10:0020 Apr 2026 at 16:5620 Apr 2026 at 10:0617 Aor 2026 at 10:16|15 Apr 2026 at 11:1414 Anr 2026 at 17:3714 Apr 2026 at 10:099 Apr 2026 at 14:479 Aor 2026 at 10:078 Apr 2026 at 10:137 Apr 2026 at 10:016 Aor 2026 at 10:082 Anr 2026 at 10:2131 Mar 2026 at 18:2931 Mar 2026 at 10:1030 Mar 2026 at 17:1227 Mar 2026 at 10:0926 Mar 2026 at 0:50l24 Mar 2026 at 10:0023 Mar 2026 at 17:0323 Mar 2026 at 10:0020 Mar 2026 at 11:4620 Mar 2026 at 10:0619 Mar 2026 at 12:0110 Mar 2026 at 11:2519 Mar 2026 at 9:5718 Mar 2026 at 16:2017 Mar 2026 at 17:4016 Mar 2026 at 16:55MG MOr 200G C+ 40:0213 Mar 2026 at 10:1212 Mar 2026 at 18:3512 Mar 2026 at 10:1011 Mar 2026 at 10:0610 Mar 2026 at 9:579 Mar 2026 at 17:04o Mor 2006 nt d.ce4 Mar 2026 at 11:097 Mar 2026 at 10:07== Colden491,3 MB1,37 GBMPEG-4 movie1.55 G:MPEG-4 movie931,7 MB MPEG-4 movie1,86 GB832,2 MEMPEG-4 movie72AMRMDSG-A movie1,74 GBMPEG-4 movie1,36 GBMPEG-4 movie1.41 G:MPEG-A movie567,8 MB MPEG-4 movie4,25 GB698,5 MBMPEG-4 movie1,16 GBMDEG.A movid513,4 MB2,75 GBMPEG-4 movie1 AAGMPEG-A movid924,4 MB MPEG-4 movie362,6 MB748.8 ME1.04 GBMPEG-4 movieMPEG-4 movie20.5 M:1.02 GP4,68 GB3,4 GB923.6 ME2,77 GBMPEG-4 movieMPEG-A movieMPEG-4 movieMPEG-4 movieMPEG-4 movieMDEC.A movid884,3 MBMPEG-4 movieA76 6 MPMDEG-A movie550,8 MBMPEG-4 movie3,44 GB438.9 MEMPEG-4 movie1,68 GBMoeeA mowid2.38 G:MPEG-4 movie2 26 GRMDSG-A movie386,3 MBMPEG-4 movie705,8 MBMPEG-4 movie2.78 GEMPEG-4 movie1,53 GB MPEG-4 movie1,2 CB4.19 GEMPEG-4 moviec022MeMoECA mAvid1,02 CBMPEG-4 movie637,6 MBMPEG-4 movie978.7 MEMPEG-A movid798.7 MBMPEG-4 movie404,6 MB4.16 GEMPEG-4 movie0107 MpMDEeA MAvid291,7 MBMPEG-4 movie2,62 GBMPEG-4 movie769 5 MPMDEG.A movic1 of 153 selected. 1.94 TB available...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15634
|
|
15633
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
FirefoxVIewHistorybookmarksProfiles Tools©github.com/jimWindow HelpE isjiminny / app &Q Type to search+.<> Code12 Pull requests 34& Agents• Actions • Wiki © Security and quality 4 ~ Insightsfs Settings@ On April 24 we'll start using GitHub Copilot interaction data for Al model training unless you opt out. Review this update and manage your preferences in your GitHub account settingsA Vasil-Jiminny requested your review on this pull request.Jy 20820 es reindex stream model hydration #12059 •I? OpenVasil-Jiminny wants to merae 36 commits into master from JY-20820-es-reindex-stream-model-hydration U• Checks pending@ Conversation 8a Commits 36E Checks 2 # Files changed 12Vasil-Jiminny commented 3 days ago • edited -JIRA: JY-208L0Chanaes:• Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data,and is then destroved. Previousiv a batch of 100 models were loaded, with a massive overhead lust tor keemina acollection of models• Move Fmail filters from I oadDocuments to db querv. Initiallv email filters were added to I oadDocuments so thatwhen we find an email for indexing, we send a "delete document" command to ES for that document. This leads to• Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is notnecessary anymore. Dropped.• Workerld is automaticallv generatedi• SkipActivityTrait is deleted.• Tests are updated.• Entitv@uervBuilder is now instantiable. Methods are called from instancel• ElasticSearchPartialUpdate is marked as deprecated.Code -+262 .140 6882lE Vasil-Jiminny and others added 28 commits 4 days ago@ Add "isDeleted" accessor methods for usage in building ES payload.a Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr... ...• a Add isDeleted to the Searchable contract.• Move the logic of Skio Emails into the elastic search activity reposi.. ..o @ Since skipping emails is now delegated to the DB, we can remove artif... ..adc64ba3b39a463ca67a0c7b2fb5470e119a7eh424Reviewersyalokin-jiminny€ vkulovLakyLakStill in progress? Convert to draftAssigneesNo one—assign yoursellNone yetProiectsNone vetMilestoneNo milestoneDevelopmentSuccessfully merging this pull request may close theseissues.None vetNotificationsUnsubscribenartcnonteravountesjiminny© RecentsA ApplicationsDocumentsii lukasiCloud• iCloud Drive288 Sync folderLocationsC DXP4800PLUS-B5F 4® Network• CRM• Orange• Red• Yellow• Green• Blue• PurpleO All Tags...workNamev N 2026- Daily 2026-05-11.mp4* Dally 2020-00-08.mp4к 1-1 2026-05-07mo4#a Daily 2026-05-07.mp4* 1-1 2026-04-24.mp4= Daily 2026-04-24.mp4m User Pilot introduction Adi 2026-04-23.mp4Daily 2026-04-23.mp4Daily 2026-04-22.mp4*m Refinement 2026-04-06.mp4Daily 2026-04-21.mp4Du Refinement 2026-04-20.mp4zz Daily 2026-04-17.mp4ww Planning 2026-04-15.mp4Retro 2026-04-14.mn/• Daily 2026-04-14.mp4User pilot (Adi) 2026-04-09.mp4• Dailv 2026-04-09,mo4w: Daily 2026-04-08.mp4• Daily 2026-04-07.mp4• Dallv 2026-04-06.mo4= Daily 2026-04-02 mn4ining 2026-04-01 & task split.mp4i: Retro 2026-03-31.mp4em Dailv 2026-03-31.mo4- Refinement 2026-03-30.mp4- Dallv 2026-03-27mo4• Daily 2026-02-26 mn4- Daily 2026-03-24.mp4= Refinement 2026-03-23.mp4• Dailv 2026-03-23.mo4•* BE chapter 2026-03-20.mp4- Daily 2026-03-20.mp4a Planina 2026-03-18-converted.mo4|- Refinement 2026-02-09-converted.mp4• Review 2026-03-18.mp4aa Planina 2026-03-18.mo4v Retro 2026-03-17.mp4- Daily 2026-03-17.mp4• Refinement 2026-03-16.mo4|- Daily 2026-03-16.mp4Daily 2026-03-13.mp4* 1-12026-03-12.mp4Daily 2026-03-12 mn/ea Daily 2026-03-11.mp4Daily 2026-03-10.mp4TE: Refineant 2026-03-09 mo4вaлail bane лe,л0 mnлl* Plannina 2026-03-04.mo4C nail 2026.02.02 mm/l$0Support Daily - in 4h 55m100% 28• Mon 11 May 10:05:32Q SeaDate ModifiedTodav at 10:048 May 2026 at 10:227 May 2026 at 18:217 May 2026 at 10:1024 Apr 2026 at 14:4424 Apr 2026 at 10:1122 Anr 2026 at 11:5923 Apr 2026 at 10:3222 Apr 2026 at 10:2171 Aor 2026 at 11:0721 Apr 2026 at 10:0020 Apr 2026 at 16:5620 Apr 2026 at 10:0617 Aor 2026 at 10:16|15 Apr 2026 at 11:1414 Anr 2026 at 17:3714 Apr 2026 at 10:099 Apr 2026 at 14:479 Aor 2026 at 10:078 Apr 2026 at 10:137 Apr 2026 at 10:016 Aor 2026 at 10:082 Anr 2026 at 10:2131 Mar 2026 at 18:2931 Mar 2026 at 10:1030 Mar 2026 at 17:1227 Mar 2026 at 10:0926 Mar 2026 at 0:50l24 Mar 2026 at 10:0023 Mar 2026 at 17:0323 Mar 2026 at 10:0020 Mar 2026 at 11:4620 Mar 2026 at 10:0619 Mar 2026 at 12:0110 Mar 2026 at 11:2519 Mar 2026 at 9:5718 Mar 2026 at 16:2017 Mar 2026 at 17:4016 Mar 2026 at 16:55MG MOr 200G C+ 40:0213 Mar 2026 at 10:1212 Mar 2026 at 18:3512 Mar 2026 at 10:1011 Mar 2026 at 10:0610 Mar 2026 at 9:579 Mar 2026 at 17:04o Mor nnnG nt 0.564 Mar 2026 at 11:097 Mar 2026 at 10:07== Colden491,3 MB1,37 GBMPEG-4 movie1.55 G:MPEG-4 movie931,7 MB MPEG-4 movie1,86 GB832,2 MEMPEG-4 movie72AMPMDSG-A movie1,74 GBMPEG-4 movie1,36 GBMPEG-4 movie1.41 G:MPEG-A movie567,8 MB MPEG-4 movie4,25 GB698,5 MBMPEG-4 movie1,16 GBMDEG.A movid513,4 MB2,75 GBMPEG-4 movie1 AAGMPEG-A movid924,4 MB MPEG-4 movie362,6 MB748.8 ME1.04 GBMPEG-4 movieMPEG-4 movie205 M-1.02 GP4,68 GB3,4 GB923.6 ME2,77 GBMPEG-4 movieMPEG-A movieMPEG-4 movieMPEG-4 movieMPEG-4 movieMDEC.A movid884,3 MBMPEG-4 movieA76 6 MPMDEG-A movie550,8 MBMPEG-4 movie3,44 GB438.9 MEMPEG-4 movie1,68 GBMoeeA mowid2.38 G:MPEG-4 movie2 26 GRMDSG-A movie386,3 MBMPEG-4 movie705,8 MBMPEG-4 movie2.78 GEMPEG-4 movie1,53 GB MPEG-4 movie1,2 CB4.19 GEMPEG-4 moviec022MeMoECA mAvid1,02 CBMPEG-4 movie637,6 MBMPEG-4 movie978.7 MEMPEG-A movid798.7 MBMPEG-4 movie404,6 MB4.16 GEMPEG-4 movie0107 MpMDEeA MAvid291,7 MBMPEG-4 movie2,62 GBMPEG-4 movie769 5 MPMDSG.A movid1 of 153 selected. 1.94 TB available...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15633
|
|
15632
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelp<DEV (docker)• жз• Support Daily - in 4h 55 mDOCKER₴1DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY]@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% <78• Mon 11 May 10:05:33181screenpipe"0 ₴6DEV...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15632
|
|
15631
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago
3 days ago
@Vasil-Jiminny
EntityQueryBuilderTest is improved.
EntityQueryBuilderTest is improved....
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15631
|
|
15630
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15630
|
|
15629
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15629
|
|
15628
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Preview
Preview
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Pull Request Toolbar
Pull Request Toolbar
Collapse file tree
Open
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Commit 098a688
Commit 098a688
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
0 of 2 files viewed
Checks pending
Checks pending
Submit review
Submit
review
Open diff view settings
Open overview panel
Open comments panel
(
0
)
Filter files…
Filter options
File tree
File tree
app/Component/ES/Processor/Actions
LoadDocumentsAction.php
LoadDocumentsAction.php
tests/Unit/Component/ES/Processor/Actions
LoadDocumentsActionTest.php
LoadDocumentsActionTest.php
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Previous commit
Prev
No next commit
Next
Browse files
Browse files
Vasil-Jiminny
commits by Vasil-Jiminny
Vasil-Jiminny
committed
3 days ago
·
Status checks: failure
8 / 10
commit
098a688
098a688
Copy full SHA for 098a688
Collapse file
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Copy file name to clipboard
Expand all lines: app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Lines changed: 2 additions & 2 deletions
Not Viewed
Viewed
Comment on this file
More options
Original file line number
Original file line
Diff line number
Diff line change
@@ -29,7 +29,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
30
30
31
/** @var Model&Searchable $entityModel */
31
/** @var Model&Searchable $entityModel */
32
-
foreach
(
$
query
->
cursor
(
)
as
$
entityModel
) {
32
+
foreach
(
$
query
->
lazyByIdDesc
(
50
,
'
id
'
)
as
$
entityModel
) {
33
if
(
$
entityModel
->
isDeleted
()) {
33
if
(
$
entityModel
->
isDeleted
()) {
34
/**
34
/**
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
@@ -54,7 +54,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
54
}
54
}...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15628
|
|
15627
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Preview
Preview
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Pull Request Toolbar
Pull Request Toolbar
Collapse file tree
Open
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Commit 098a688
Commit 098a688
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
0 of 2 files viewed
Checks pending
Checks pending
Submit review
Submit
review
Open diff view settings
Open overview panel
Open comments panel
(
0
)
Filter files…
Filter options
File tree
File tree
app/Component/ES/Processor/Actions
LoadDocumentsAction.php
LoadDocumentsAction.php
tests/Unit/Component/ES/Processor/Actions
LoadDocumentsActionTest.php
LoadDocumentsActionTest.php
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Previous commit
Prev
No next commit
Next
Browse files
Browse files
Vasil-Jiminny
commits by Vasil-Jiminny
Vasil-Jiminny
committed
3 days ago
·
Status checks: failure
8 / 10
commit
098a688
098a688
Copy full SHA for 098a688
Collapse file
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Copy file name to clipboard
Expand all lines: app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Lines changed: 2 additions & 2 deletions
Not Viewed
Viewed
Comment on this file
More options
Original file line number
Original file line
Diff line number
Diff line change
@@ -29,7 +29,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
30
30
31
/** @var Model&Searchable $entityModel */
31
/** @var Model&Searchable $entityModel */
32
-
foreach
(
$
query
->
cursor
(
)
as
$
entityModel
) {
32
+
foreach
(
$
query
->
lazyByIdDesc
(
50
,
'
id
'
)
as
$
entityModel
) {
33
if
(
$
entityModel
->
isDeleted
()) {
33
if
(
$
entityModel
->
isDeleted
()) {
34
/**
34
/**
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
@@ -54,7 +54,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15627
|
|
15626
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Preview
Preview
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Pull Request Toolbar
Pull Request Toolbar
Collapse file tree
Open
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Commit 098a688
Commit 098a688
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
0 of 2 files viewed
Checks pending
Checks pending
Submit review
Submit
review
Open diff view settings
Open overview panel
Open comments panel
(
0
)
Filter files…
Filter options
File tree
File tree
app/Component/ES/Processor/Actions
LoadDocumentsAction.php
LoadDocumentsAction.php
tests/Unit/Component/ES/Processor/Actions
LoadDocumentsActionTest.php
LoadDocumentsActionTest.php
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Usage of cursor introduced more problems - mainly no eager loading and keeping DB connection open and busy. For that reason it is swapped by LazyByIdDesc. Smaller batches are loaded in memory, cleaned up one by one. More smaller batches are loaded in general. If smaller batches help keep memory lower (at the cost of more smaller db requests), we will load more data, in smaller chunks.
Previous commit
Prev
No next commit
Next
Browse files
Browse files
Vasil-Jiminny
commits by Vasil-Jiminny
Vasil-Jiminny
committed
3 days ago
·
Status checks: failure
8 / 10
commit
098a688
098a688
Copy full SHA for 098a688
Collapse file
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Copy file name to clipboard
Expand all lines: app/Component/ES/Processor/Actions/LoadDocumentsAction.php
Lines changed: 2 additions & 2 deletions
Not Viewed
Viewed
Comment on this file
More options
Original file line number
Original file line
Diff line number
Diff line change
@@ -29,7 +29,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
29
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
30
30
31
/** @var Model&Searchable $entityModel */
31
/** @var Model&Searchable $entityModel */
32
-
foreach
(
$
query
->
cursor
(
)
as
$
entityModel
) {
32
+
foreach
(
$
query
->
lazyByIdDesc
(
50
,
'
id
'
)
as
$
entityModel
) {
33
if
(
$
entityModel
->
isDeleted
()) {
33
if
(
$
entityModel
->
isDeleted
()) {
34
/**
34
/**
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
35
* Cleanup (from ElasticSearch) scheduled entities that were recently deleted.
@@ -54,7 +54,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
54
}
54
}
55
55
56
/**
56
/**
57
-
* Clean up
fragmented
memory.
57
+
* Clean up
associated model
memory.
58
* Dropping relations and unsetting the entity model after hydration and usage,
58
* Dropping relations and unsetting the entity model after hydration and usage,
59
* allows GC to remove heap memory allocations, and recycle already allocated memory,
59
* allows GC to remove heap memory allocations, and recycle already allocated memory,
60
* instead of allocating more memory from OS.
60
* instead of allocating more memory from OS.
Add comment
More actions
Original file line number
@@ -29,7 +29,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
29
30
31
32
33
34
35
@@ -54,7 +54,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
54
55
56
57
58
59
60
Original file line
@@ -29,7 +29,7 @@ public function loadDocuments(string $updateTarget, array $entityIdsList): Docum
$
query
=
$
this
->
queryBuilder
->
getEntityQuery
(
$
updateTarget
,
$
entityIdsList
);
/** @var Model&Searchable $entityModel */
-
foreach
(
$
query
->
cursor
(
)
as
$
entityModel
) {
if
(
$
entityModel
->
isDeleted
()) {...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059/changes/098a6883 github.com/jiminny/app/pull/12059/changes/098a68836c1c00a69ae30dce317463511f1145a3...
|
15626
|
|
15625
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15625
|
|
15624
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15624
|
|
15623
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15623
|
|
15622
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago
3 days ago
@Vasil-Jiminny
EntityQueryBuilderTest is improved.
EntityQueryBuilderTest is improved.
8 / 10 checks OK
851b70d
851b70d
@Vasil-Jiminny
Explain why "runForSeconds" cannot be simply removed.
Explain why "runForSeconds" cannot be simply removed.
e25d546
e25d546
@Vasil-Jiminny
Call methods internally, without "static".
Call methods internally, without "static".
e53f9cf
e53f9cf
@Vasil-Jiminny
Better comments.
Better comments.
12 / 12 checks OK
c326474
c326474
@Vasil-Jiminny
Show options
Author
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15622
|
|
15621
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago
3 days ago
@Vasil-Jiminny
EntityQueryBuilderTest is improved.
EntityQueryBuilderTest is improved.
8 / 10 checks OK
851b70d
851b70d
@Vasil-Jiminny
Explain why "runForSeconds" cannot be simply removed.
Explain why "runForSeconds" cannot be simply removed.
e25d546
e25d546
@Vasil-Jiminny
Call methods internally, without "static".
Call methods internally, without "static".
e53f9cf
e53f9cf
@Vasil-Jiminny
Better comments.
Better comments.
12 / 12 checks OK
c326474
c326474
@Vasil-Jiminny
Show options
Author
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15621
|
|
15620
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Open
Jy 20820 es reindex stream model hydration #12059 Vasil-Jiminny wants to merge 36 commits into master from JY-20820-es-reindex-stream-model-hydration Copy head branch name to clipboard
Jy 20820 es reindex stream model hydration
Jy 20820 es reindex stream model hydration
#
12059
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago
3 days ago
@Vasil-Jiminny
EntityQueryBuilderTest is improved.
EntityQueryBuilderTest is improved.
8 / 10 checks OK
851b70d
851b70d
@Vasil-Jiminny
Explain why "runForSeconds" cannot be simply removed.
Explain why "runForSeconds" cannot be simply removed.
e25d546
e25d546
@Vasil-Jiminny
Call methods internally, without "static".
Call methods internally, without "static".
e53f9cf
e53f9cf
@Vasil-Jiminny
Better comments.
Better comments.
12 / 12 checks OK
c326474
c326474
@Vasil-Jiminny
Show options
Author
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
Comments 2 and 3 are resolved
Regarding comment 1 - There is currently a supervisor config that starts the command with the option provided. If the option is removed from the command signature, the command throws an error and will not start. When the supervisor config is updated, we can remove the option as well, right now it should remain as is.
Comment 4 - the generator returns an instance of Model & Searchable. Model provides access to
setRelations
method.
Comment 5 - SentryMock is within the namespace of the test class.
Add or remove reactions
react with eyes
👀
1...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15620
|
|
15619
|
New Tab
Close tab
Jy 20820 es reindex stream model New Tab
Close tab
Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Skip to content
Skip to content
Open menu
Homepage (g then d)
jiminny
jiminny
app
app
Search or jump to…
Type
/
to search
Chat with Copilot
Open Copilot…
Create new...
All issues(g then i)
All pull requests
All repositories
You have unread notifications(g then n)
Open user navigation menu
Repository navigation
Repository navigation
Code
Code
Pull requests (34)
Pull requests
(
34
)
Agents
Agents
Actions
Actions
Wiki
Wiki
Security and quality (4)
Security and quality
(
4
)
Insights
Insights
Settings
Settings
Important update
Important update
On April 24 we'll start using GitHub Copilot interaction data for AI model training unless you opt out.
Review this update
Review this update
and manage your preferences in your
GitHub account settings
GitHub account settings
.
Dismiss banner
Review requested
Review requested
Vasil-Jiminny
Vasil-Jiminny
requested your review on this pull request.
Add your review
Add your review
Jy 20820 es reindex stream model hydration #12059 Edit title
Jy 20820 es reindex stream model hydration
#
12059
Edit title
Checks pending
Checks pending
Code
Code
Open
Vasil-Jiminny
Vasil-Jiminny
wants to merge 36 commits into
master
master
from
JY-20820-es-reindex-stream-model-hydration
JY-20820-es-reindex-stream-model-hydration
Copy head branch name to clipboard
Lines changed: 363 additions & 140 deletions
Conversation (8)
Conversation
(
8
)
Commits (36)
Commits
(
36
)
Checks (2)
Checks
(
2
)
Files changed (12)
Files changed
(
12
)
Conversation
Conversation
@Vasil-Jiminny
Show options
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
JIRA: JY-20820
JIRA:
JY-20820
JY-20820
Changes:
Changes:
Load documents for reindexing by streaming raw data into a single model, that is hydrated, extracts indexing data, and is then destroyed. Previously a batch of 100 models were loaded, with a massive overhead just for keeping a collection of models
Move Email filters from LoadDocuments to db query. Initially email filters were added to LoadDocuments, so that when we find an email for indexing, we send a "delete document" command to ES for that document. This leads to self-cleanup over time. Now it's unnecessary.
Since introducing 'es-update' worker group with long running processes, start command 'runForSeconds' is not necessary anymore. Dropped.
WorkerId is automatically generated.
SkipActivityTrait is deleted.
Tests are updated.
EntityQueryBuilder is now instantiable. Methods are called from instance.
ElasticSearchPartialUpdate is marked as deprecated.
Add or remove reactions
Vasil-Jiminny
Vasil-Jiminny
and others
added
28
commits
4 days ago
4 days ago
@Vasil-Jiminny
Add "isDeleted" accessor methods for usage in building ES payload.
Add "isDeleted" accessor methods for usage in building ES payload.
adc04ba
adc04ba
@Vasil-Jiminny
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
Remove method "isDeleted" from ActivityElasticSearchTrait as it's alr…
…
3b39a46
3b39a46
@Vasil-Jiminny
Add isDeleted to the Searchable contract.
Add isDeleted to the Searchable contract.
3ca67a0
3ca67a0
@Vasil-Jiminny
Move the logic of Skip Emails into the elastic search activity reposi…
Move the logic of Skip Emails into the elastic search activity reposi…
…
c7b2fb5
c7b2fb5
@Vasil-Jiminny
Since skipping emails is now delegated to the DB, we can remove artif…
Since skipping emails is now delegated to the DB, we can remove artif…
…
470e119
470e119
@Vasil-Jiminny
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
Refactor LoadActivityAction to use a cursor iterator over objects. Me…
…
a7eb43f
a7eb43f
@Vasil-Jiminny
SkipActivityTrait and corresponding test are now deleted.
SkipActivityTrait and corresponding test are now deleted.
0ed71ff
0ed71ff
@Vasil-Jiminny
Code style improvement.
Code style improvement.
79e7ee3
79e7ee3
@Vasil-Jiminny
Added comments explaining the changes in Load Documents.
Added comments explaining the changes in Load Documents.
b6e9c6d
b6e9c6d
@Vasil-Jiminny
Minor code style changes.
Minor code style changes.
07fa98c
07fa98c
@Vasil-Jiminny
EntityQueryBuilderTest is updated to not use static methods.
EntityQueryBuilderTest is updated to not use static methods.
565a571
565a571
@Vasil-Jiminny
"Final readonly" restrictions removed from EntityQueryBuilder class.
"Final readonly" restrictions removed from EntityQueryBuilder class.
46590e5
46590e5
@Vasil-Jiminny
Make EntityQuertBuilder instantiable.
Make EntityQuertBuilder instantiable.
6d34ebe
6d34ebe
@Vasil-Jiminny
LoadDocumentsAction instantiates the queryBuilder.
LoadDocumentsAction instantiates the queryBuilder.
27277b8
27277b8
@Vasil-Jiminny
LoadDocumentsActions covered with a unit test (finally).
LoadDocumentsActions covered with a unit test (finally).
8 / 10 checks OK
e2693eb
e2693eb
@Vasil-Jiminny
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
Add an explanation why ElasticSearchDocumentPartialUpdater should not…
…
6da0165
6da0165
@Vasil-Jiminny
Better testing mechanism for Sentry.
Better testing mechanism for Sentry.
8 / 10 checks OK
80a2057
80a2057
@Vasil-Jiminny
Fix a test issue in EntityQueryBuilderTest
Fix a test issue in EntityQueryBuilderTest
12 / 12 checks OK
46330fa
46330fa
@Vasil-Jiminny
Allow selection of chunks with up to 500 items.
Allow selection of chunks with up to 500 items.
21 / 21 checks OK
f7c53fa
f7c53fa
@Vasil-Jiminny
Drop the "ID" option from update-entities command. WorkerId will be a…
Drop the "ID" option from update-entities command. WorkerId will be a…
…
8273321
8273321
@Vasil-Jiminny
Directly use the stringified UUID instead of the object.
Directly use the stringified UUID instead of the object.
6f50572
6f50572
@Vasil-Jiminny
Drop the "runForSeconds" paramter.
Drop the "runForSeconds" paramter.
4edceca
4edceca
@Vasil-Jiminny
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
Drop the "runForSeconds" paramter. Run a command until "shouldStop" o…
…
ed12e2b
ed12e2b
@Vasil-Jiminny
Partially restore the "runForSeconds" attribute, until it's removed f…
Partially restore the "runForSeconds" attribute, until it's removed f…
…
21 / 21 checks OK
f029263
f029263
@Vasil-Jiminny
Revert chunk size changes.
Revert chunk size changes.
21 / 21 checks OK
6ed9e2b
6ed9e2b
@Vasil-Jiminny
Remove an unnecessary whitespace.
Remove an unnecessary whitespace.
9acb137
9acb137
@Vasil-Jiminny
Merge branch '
Merge branch '
JY-20808
JY-20808
-low-priority-indexing-queue' into
-low-priority-indexing-queue' into
JY-20820
JY-20820
-es-…
-es-…
…
8 / 10 checks OK
37241b6
37241b6
@Vasil-Jiminny
Merge branch 'master' into
Merge branch 'master' into
JY-20820
JY-20820
-es-reindex-stream-model-hydration
-es-reindex-stream-model-hydration
Verified
Verified
4 / 6 checks OK
b2f4fc9
b2f4fc9
@Vasil-Jiminny
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
@claude
This comment was marked as outdated. Show comment
This comment was marked as outdated.
This comment was marked as outdated.
Show comment
Vasil-Jiminny
Vasil-Jiminny
added
4
commits
3 days ago
3 days ago
@Vasil-Jiminny
EntityQueryBuilderTest is improved.
EntityQueryBuilderTest is improved.
8 / 10 checks OK
851b70d
851b70d
@Vasil-Jiminny
Explain why "runForSeconds" cannot be simply removed.
Explain why "runForSeconds" cannot be simply removed.
e25d546
e25d546
@Vasil-Jiminny
Call methods internally, without "static".
Call methods internally, without "static".
e53f9cf
e53f9cf
@Vasil-Jiminny
Better comments.
Better comments.
12 / 12 checks OK
c326474
c326474
@Vasil-Jiminny
Show options
Author
Vasil-Jiminny commented 3 days ago •
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago
•
edited
edited
Comments 2 and 3 are resolved
Regarding comment 1 - There is currently a supervisor config that starts the command with the option provided. If the option is removed from the command signature, the command throws an error and will not start. When the supervisor config is updated, we can remove the option as well, right now it should remain as is.
Comment 4 - the generator returns an instance of Model & Searchable. Model provides access to
setRelations
method.
Comment 5 - SentryMock is within the namespace of the test class.
Add or remove reactions
react with eyes
👀
1
@Vasil-Jiminny
Show options
Author
Vasil-Jiminny commented 3 days ago
Vasil-Jiminny
Vasil-Jiminny
commented
3 days ago
3 days ago...
|
Firefox
|
Jy 20820 es reindex stream model hydration by Vasi Jy 20820 es reindex stream model hydration by Vasil-Jiminny · Pull Request #12059 · jiminny/app — Work...
|
github.com/jiminny/app/pull/12059
|
15619
|
|
15618
|
New Tab
Close tab
github.com/jiminny/app/pull/1205 New Tab
Close tab
github.com/jiminny/app/pull/12059
Close tab
New Tab
Open Google Gemini (⌃X)
Tabs from other devices
Open history (⇧⌘H)
Open bookmarks (⌘B)
Customize sidebar
Waiting for github.com…
FirefoxcalHistorybookmarksProtllesToolsWindowmelpgithub.com/jiminny/app/pull/12059workFavourites• jiminny(®) AirDrop• Recents# Application|9 Documents• Downloadsii lukasiCloud• iCloud Drive999 Svnc toldeLocations0J DXP4800PLUS-B5F A49 Network• CRM• Orange• Red• Yellow• Greer• Purple• All Tags..v 2026= Daily 2026-05-11.mp4* Dally 2020-00-08.mp4к 1-1 2026-05-07mo4#a Daily 2026-05-07.mp4wя 1-1 2026-04-24.mp4= Daily 2026-04-24.mp4m User Pilot introduction Adi 2026-04-23.mp4Daily 2026-04-23.mp4Daily 2026-04-22.mp4*m Refinement 2026-04-06.mp4= Daily 2026-04-21.mp4Di Refinement 2026-04-20.mp4Daily 2026-04-20.mp4ta Daily 2026-04-17.mp4ww Planning 2026-04-15.mp4Retro 2026-04-14.mn4In Daily 2026-04-14.m04= User pilot (Adi) 2026-04-09.mp4• Dailv 2026-04-09,mo4wa Daily 2026-04-08.mp4• Daily 2026-04-07.mp4• Dallv 2026-04-06.mo4= Daily 2026-04-02 mn4lax Plannina 2026-04-01 & task split.mp4Retro 2026-03-31.mp4em Dailv 2026-03-31.mo4• Refinement 2026-03-30.mp4- Dallv 2026-03-27mo4• Daily 2026-02-26 mn4• Dailv 2026-03-24.mp4• Refinement 2026-03-23.mp4• Dailv 2026-03-23.mo4** BE chapter 2026-03-20.mp4Daily 2026-03-20.mp4a Planina 2026-03-18-converted.mo4• Refinement 2026-02-09-converted.mp4• Review 2026-03-18.mp4aa Planina 2026-03-18.mo4F Retro 2026-03-17.mp4- Daily 2026-03-17.mp4•: Refinement 2026-03-16.mр4Daily 2026-03-16.mp4ra 1-1 2026-03-12.mp4Daily 2026-03-12 mn/na. Daily 2026-03-11.mр4-: Daily 2026-03-10.mp4TE: Refineant 2026-03-09 mo4вaлail bane лe,л0 mnлl* Plannina 2026-03-04.mo4C nail 2026.02.02 mm/l1 Daily - Platform • 1 m leftQ SeaiDate ModifiedTodav at 10:048 May 2026 at 10:227 May 2026 at 18:217 May 2026 at 10:1024 Apr 2026 at 14:4424 Apr 2026 at 10:1122 Anr 2026 at 11:5923 Apr 2026 at 10:3222 Apr 2026 at 10:2171 Aor 2026 at 11:0721 Apr 2026 at 10:0020 Apr 2026 at 16:5€20 Apr 2026 at 10:0617 Aor 2026 at 10:16|15 Apr 2026 at 11:1414 Anr 2026 at 17:3714 Apr 2026 at 10:099 Apr 2026 at 14:479 Aor 2026 at 10:07Q Anr 2026 at 10:127 Apr 2026 at 10:016 Aor 2026 at 10:082 Anr 2026 at 10:2131 Mar 2026 at 18:2931 Mar 2026 at 10:1020 Mar 2026 at 17:1227 Mar 2026 at 10:0926 Mar 2026 at 0:50l24 Mar 2026 at 10:0023 Mar 2026 at 10:0020 Mar 2026 at 11:4620 Mar 2026 at 10:0619 Mar 2026 at 12:0110 Mar 2026 at 11:2519 Mar 2026 at 9:5718 Mar 2026 at 16:2018 Mar 2026 at 11:14147 MAr GA0G M 17:1016 Mar 2026 at 16:55lMG MOr 200G C+ 40:0213 Mar 2026 at 10:1212 Mar 2026 at 18:3512 Mar 2026 at 10:1011 Mar 2026 at 10:0610 Mar 2026 at 9:519 Mar 2026 at 17:04o Mor 2006 nt d.ce4 Mar 2026 at 11:097 Mar 2026 at 10:07100% 2• Mon 11 May 10:04:49Colden491.3 MB1,37 GBMPEG-4 movie1.55 G:MPEG-4 movie931,7 MB MPEG-4 movie1,86 GB832,2 MBMPEG-4 movieMDSG-A movie72AMR1,74 GB1,36 GBMPEG-4 movieMPEG-4 movie1.41 G:MPEG-A movie567.8 MB MPEG-4 movie4,25 GBMPEG-4 movie698,5 MB1 16 GRMPEG-4 movieMDEG.A movid2,75 GB1 AAGMPEG-4 movieMPEG-A movid924.4 MB MPEG-4 movie362,6 MB748.8 MEMPEG-4 movie1oл epMDEG.A movid205 M-MPEG-4 movie1.02 GP4,68 GB3,4 GB923.6 MEMPEG-A movieMPEG-4 movieMPEG-4 movieMPEG-4 movie2,77 GBMDEC.A movid884.3 М:MPEG-4 movieA76 6 MP550,8 MB3.44 GBMDEG-A movieMPEG-4 movieMPEG-4 movie438.9 MEMPEG-4 movie1.68 GBMoeeA mavir2.38 G:MPEG-4 movie2 26 GRMDSG-A movie386,3 MBMPEG-4 movie705.8 MBMPEG-4 movie2.78 GEMPEG-4 movie1,53 GB MPEG-4 movie1,2 CB4.19 GEMPEG-4 moviec022Me1,02 GB637.6 MEMoECA mAvidMPEG-4 movieMPEG-4 movie978.7 ME900 9410404,6 MB4.16 GEMPEG-A moviduoee hmadlMPEG-4 movie0107 Mp291,7 MB2.62 GEMDEeA MAvidMDEG_A movieMPEG-4 movie769 5 MPMDEG.A movic4o€ 420 calaatad A OA TO aucilahl)Waitina for aithub.com...
|
Firefox
|
Work — Mozilla Firefox
|
github.com/jiminny/app/pull/12059
|
15618
|
|
15617
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Toast
Jira Cloud
Google Calendar
Messages
Messages
Add canvas
Add canvas
Files
Files
Pins
Pins
Add and Edit Channel Tabs
Canvas
List
Folder
Apr 28th at 4:49:02 PM
4:49
Никога не ми се е налагало да работя с тоя Postmark
Lukas Kovalik
Apr 28th at 4:49:27 PM
4:49 PM
то по скоро Amazon credentials ми е въпрос
Vasil Vasilev
Apr 28th at 4:51:31 PM
4:51 PM
a, ти искаш в amazon да добавим ключ за достъп до QAi ?
Apr 28th at 4:51:38 PM
4:51
Вес се грижи за тея неща
Apr 28th at 4:52:06 PM
4:52
дори не съм сигурен дали тоя ключ не трябва да бъде в CircleCI при билда на имиджа
Apr 28th at 4:52:10 PM
4:52
т.е.
Apr 28th at 4:52:12 PM
4:52
пак не знам
Lukas Kovalik
Apr 28th at 4:53:41 PM
4:53 PM
ок, ще питам Вес, мерси
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Vasil Vasilev
Apr 28th at 5:00:16 PM
5:00 PM
моля
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Jump to date
Vasil Vasilev
May 8th at 2:52:43 PM
2:52 PM
Лукаш, привет
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:52:48 PM
2:52
хвърли моля те едно око тука
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:52:49 PM
2:52
https://github.com/jiminny/app/pull/12059
https://github.com/jiminny/app/pull/12059
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:53:03 PM
2:53
опитвам се да оптимизирам процеса по индексиране на активитита за ЕС
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:54:16 PM
2:54
идеята е да намаля паметта която се ползва за да се генерира един бач от 100 активитита
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:54:35 PM
2:54
и после да увелича размера на бачовете, за да имаме по малко blocking операции в ЕС, като реиндексира
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Lukas Kovalik
May 8th at 4:12:58 PM
4:12 PM
здрасти, изглежда ок, но когато го минах и през gemini ми даде един warning.
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:13:17 PM
4:13
Switch
cursor()
to
lazyById(250)
. It preserves the single-loop, generator-style code in the new version while restoring proper batched eager loading (avoiding N+1 on
getIndexableAttributes()
) and releasing the DB connection between chunks (avoiding long-held PDO connections during ES/Sentry calls).
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
slackcalVIewJiminny…..HomeActivityFllesLaterMore# curiosity_lab# engineeringi generall# jiminny-bg# platform-tickets# product_launches# randomi released# sofia-office# support# thank-yous# the_people_of jimi...• Direct messages€. Vasil VasilevC. Nikolay IvanovP. Galya Dimitrova3 Aneliya Angelova, ...8o Stoyan Tanev8 Stefka Stovanova@ VesA. Aneliya AngelovaL James Graham. Lukas Kovalik y...i Apps® ToastSi Jira GloudmistonWindowhelp@ Describe what you are looking forC. Vasil Vasilev• Messagest Add canvasQ Files& PinsLukas Kovalik 4:53 PM0 ок, ще питам Вес. мерсиVasil Vasilev 5:00 PMFriday, May 8th~Vasil Vasilev 2:52 PMЛукаш, приветі, https://github.com/jiminny/app/pull/120592:52 https://github.com//iminny/app/pull/12059опитвам се да оптимизирам процеса поинпексипане на активитита за F6идеята е да намаля паметта която сеползва за да се генерира един бач от 100активититаи после да увелича размера на бачовете,за да имаме по малко blocking операциивFс като пеинпексипаLukas Kovallk 4:12 PMздрасти, изглежда ок, но когато го минахи през gemini ми даде един warning.Switch cursor to lazyById(250). Itnreserves the single-oon. generator-stvlecode in the new version while restoringnroner hatched enger oadino avoidino N:-1lMessage vasil vasilev+ Aa ICircle CI & PROD US8 Staging al SentryPull requests -fjimin...FirefoxddressMinbox (1 620) -lukas.kovalik…E Workers | Datadog M Jiminny Mail Dashboards | Datad...FavouritesE jiminny(* AirDrop• Recents* Applications© Documents© Downloadsii lukasiCloud• iCloud Drive999 Svnc toldeLocationsO DXP4800PLUS-B5F A® Network• CRM• Orange• Red• YellowGreen• PurpleO All Tags...liminny1Y-20542 addAJ reports.Meet - Dailv-Plattormworkv N 2026- Daily 2026-05-11.mp4* Dally 2020-00-08.mp4к 1-1 2026-05-07mo4#a Daily 2026-05-07.mp4* 1-1 2026-04-24.mp4= Daily 2026-04-24.mp4m User Pilot introduction Adi 2026-04-23.mp4Daily 2026-04-23.mp4Daily 2026-04-22.mp4*m Refinement 2026-04-06.mp4Daily 2026-04-21.mp4Diu Refinement 2026-04-20.mp4Daily 2026-04-20.mp4zz Daily 2026-04-17.mp4ww Planning 2026-04-15.mp4Retro 2026-04-14.mn/• Daily 2026-04-14.mp4= User pilot (Adi) 2026-04-09.mp4• Dailv 2026-04-09,mo4w: Daily 2026-04-08.mp4• Daily 2026-04-07.mp4• Dallv 2026-04-06.mo4= Daily 2026-04-02 mn4• Planning 2026-04-01 & task split.mp4i: Retro 2026-03-31.mp4em Dailv 2026-03-31.mo4- Refinement 2026-03-30.mp4- Dallv 2026-03-27mo4• Daily 2026-02-26 mn4Refinement 2026-03-23.mp4• Dailv 2026-03-23.mo4•* BE chapter 2026-03-20.mp4Daily 2026-03-20.mp4a Planina 2026-03-18-converted.mo4• Refinement 2026-02-09-converted.mp4- Review 2026-03-18.mp4aa Planina 2026-03-18.mo4n.• Retro 2026-03-17.mp4- Daily 2026-03-17.mp4•: Refinement 2026-03-16.mр4Daily 2026-03-16.mp4* 1-12026-03-12.mp4Daily 2026-03-12 mn/a Daily 2026-03-11.mp4- Daily 2026-03-10.mp4TE: Refineant 2026-03-09 mo4вaлail bane лe,л0 mnлlas. Plannina 2026-03-04.mp4C nail 2026.02.02 mm/l| Daily - Platform - 1m left100% 28• Mon 11 May 10:04:48Q SeaiDate ModifiedTodav at 10:048 May 2026 at 10:227 May 2026 at 18:217 May 2026 at 10:1024 Apr 2026 at 14:4424 Apr 2026 at 10:1123 Apr 2026 at 11:5823 Apr 2026 at 10:3222 Apr 2026 at 10:2171 Aor 2026 at 11:0721 Apr 2026 at 10:0020 Apr 2026 at 16:5620 Apr 2026 at 10:0617 Apr 2026 at 10:1615 Apr 2026 at 11:1414 Anr 2026 at 17:3714 Apr 2026 at 10:099 Apr 2026 at 14:479 Aor 2026 at 10:078 Apr 2026 at 10:137 Apr 2026 at 10:016 Aor 2026 at 10:082 Anr 2026 at 10:2131 Mar 2026 at 18:2931 Mar 2026 at 10:1030 Mar 2026 at 17:1227 Mar 2026 at 10:0926 Mar 2026 at 0:50l24 Mar 2026 at 10:0023 Mar 2026 at 10:0020 Mar 2026 at 11:4620 Mar 2026 at 10:0619 Mar 2026 at 12:0110 Mar 2026 at 11:2519 Mar 2026 at 9:5718 Mar 2026 at 16:2018 Mar 2026 at 11:14117 Mar 2026 at 17:4016 Mar 2026 at 16:55l46 Mor 2006 6+ 40:0913 Mar 2026 at 10:1212 Mar 2026 at 18:3512 Mar 2026 at 10:1011 Mar 2026 at 10:0610 Mar 2026 at 9:579 Mar 2026 at 17:04o Mor 2006 nt d.ce4 Mar 2026 at 11:097 Mar 2026 at 10:07Colden491,3 MB1,37 GBMPEG-4 movie1.55 G:MPEG-4 movie931,7 MB MPEG-4 movie1,86 GB832,2 MBMPEG-4 movie72AMRMDSG-A movie1,74 GBMPEG-4 movie1,36 GBMPEG-4 movie1.41 G:MPEG-A movie567,8 MB MPEG-4 movie4,25 GBMPEG-4 movie698,5 MBMPEG-4 movie1,16 GBMDEG.A movid2,75 GBMPEG-4 movie1 AAGMPEG-A movid924,4 MB MPEG-4 movie362,6 MB748.8 ME1.04 GBMPEG-4 movieMPEG-4 movie205 M-1.02 GP4,68 GB3,4 GB923.6 ME2,77 GBMPEG-4 movieMPEG-A movieMPEG-4 movieMPEG-4 movieMPEG-4 movieMPEG-4 movie884.3 М:MPEG-4 movieA76 6 MPMDEG-A movie550,8 MBMPEG-4 movie3,44 GBMPEG-4 movie438.9 MEMPEG-4 movie1,68 GBMPEG-4 movie2.38 G:MPEG-4 movie2 26 GRMDSG-A movie386,3 MBMPEG-4 movie705,8 MBMPEG-4 movie2.78 GEMPEG-4 movie1,53 GB MPEG-4 movie1,2 CB4.19 GEMPEG-4 moviec022MeMoECA mAvid1,02 CBMPEG-4 movie637,6 MBMPEG-4 movie978.7 MEMPEG-A movid798.7 MBMPEG-4 movie404,6 MB4.16 GEMPEG-4 movie0107 MpMDEeA MAvid291,7 MBMPEG-4 movie2,62 GBMPEG-4 movie769 5 MPMDSG.A movid1 of 153 selected. 1.94 TB available...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15617
|
|
15616
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Toast
Jira Cloud
Google Calendar
Messages
Messages
Add canvas
Add canvas
Files
Files
Pins
Pins
Add and Edit Channel Tabs
Canvas
List
Folder
Apr 28th at 4:49:02 PM
4:49
Никога не ми се е налагало да работя с тоя Postmark
Lukas Kovalik
Apr 28th at 4:49:27 PM
4:49 PM
то по скоро Amazon credentials ми е въпрос
Vasil Vasilev
Apr 28th at 4:51:31 PM
4:51 PM
a, ти искаш в amazon да добавим ключ за достъп до QAi ?
Apr 28th at 4:51:38 PM
4:51
Вес се грижи за тея неща
Apr 28th at 4:52:06 PM
4:52
дори не съм сигурен дали тоя ключ не трябва да бъде в CircleCI при билда на имиджа
Apr 28th at 4:52:10 PM
4:52
т.е.
Apr 28th at 4:52:12 PM
4:52
пак не знам
Lukas Kovalik
Apr 28th at 4:53:41 PM
4:53 PM
ок, ще питам Вес, мерси
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Vasil Vasilev
Apr 28th at 5:00:16 PM
5:00 PM
моля
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Jump to date
Vasil Vasilev
May 8th at 2:52:43 PM
2:52 PM
Лукаш, привет
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:52:48 PM
2:52
хвърли моля те едно око тука
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:52:49 PM
2:52
https://github.com/jiminny/app/pull/12059
https://github.com/jiminny/app/pull/12059
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:53:03 PM
2:53
опитвам се да оптимизирам процеса по индексиране на активитита за ЕС
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:54:16 PM
2:54
идеята е да намаля паметта която се ползва за да се генерира един бач от 100 активитита
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 2:54:35 PM
2:54
и после да увелича размера на бачовете, за да имаме по малко blocking операции в ЕС, като реиндексира
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Lukas Kovalik
May 8th at 4:12:58 PM
4:12 PM
здрасти, изглежда ок, но когато го минах и през gemini ми даде един warning.
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
May 8th at 4:13:17 PM
4:13
Switch
cursor()
to
lazyById(250)
. It preserves the single-loop, generator-style code in the new version while restoring proper batched eager loading (avoiding N+1 on
getIndexableAttributes()
) and releasing the DB connection between chunks (avoiding long-held PDO connections during ES/Sentry calls).
React with white_check_mark
React with eyes
React with raised_hands
Add reaction…
Reply in thread
Forward message…
Save for later
More actions
Vasil Vasilev
May 8th at 4:40:06 PM
4:40 PM
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelpla6lDaily - Platform - 1 m leftDEV (docker)• жзDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:04:48181screenpipe™0 ₴6DEV...
|
Slack
|
Vasil Vasilev (DM) - Jiminny Inc - 3 new items - S Vasil Vasilev (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15616
|
|
15615
|
Switch workspaces… (Jiminny Inc) Has new messages
Switch workspaces… (Jiminny Inc) Has new messages
Home
Home
DMs
DMs
Activity
Activity
Files
Files
Later
Later
More…
More
Unreads
Threads
Huddles
Drafts & sent
1
Directories
jiminny-x-integration-app
platform-inner-team
ai-chapter
alerts
backend
bugs
confusion-clinic
curiosity_lab
engineering
general
jiminny-bg
platform-tickets
product_launches
random
releases
sofia-office
support
thank-yous
the_people_of_jiminny
Vasil Vasilev
Nikolay Ivanov
Galya Dimitrova
Aneliya Angelova
,
Nikolay Yankov
,
Steliyan Georgiev
Stoyan Tanev
Stefka Stoyanova
Ves
Aneliya Angelova
James Graham
Lukas Kovalik
you
Jira Cloud
Toast
Google Calendar
Home
Messages
Messages
About
Jump to date
Jira Cloud
APP
Apr 30th at 5:52:26 PM
5:52 PM
@Nikolay Ivanov
@Nikolay Ivanov
transitioned a Bug you are watching from
In Dev
⟶
Code Review
JY-20699 Wrong formatting for summary in the CRM
JY-20699 Wrong formatting for summary in the CRM
Status:
Code Review
Type:
Bug
Assignee:
Nikolay Ivanov
Priority:
Medium
Comment
Comment
More actions...
Apr 30th at 5:52:50 PM
5:52
@Nikolay Ivanov
@Nikolay Ivanov
commented on a Bug you are watching
JY-20699 Wrong formatting for summary in the CRM
JY-20699 Wrong formatting for summary in the CRM
Before vs After
Comment
Comment
More actions...
Jump to date
Jira Cloud
APP
May 4th at 11:54:41 AM
11:54 AM
@Nikolay Ivanov
@Nikolay Ivanov
transitioned a Bug you are watching from
Code Review
⟶
Deployed
JY-20699 Wrong formatting for summary in the CRM
JY-20699 Wrong formatting for summary in the CRM
Status:
Deployed
Type:
Bug
Assignee:
Nikolay Ivanov
Priority:
Medium
Comment
Comment
More actions...
Jump to date
New
Jira Cloud
APP
Today at 9:48:55 AM
9:48 AM
Nikolay Yankov transitioned a Bug you are assigned to from
Code Review
⟶
Deployed
JY-20818 Move Ask Jiminny reports to separated datadog metric
JY-20818 Move Ask Jiminny reports to separated datadog metric
Status:
Deployed
Type:
Bug
Assignee:
Lukas
Kovalik
Priority:
Medium
Comment
Comment
More actions...
loading…
Channel
Firefox FileEditViewHistoryBookmarksProfilesToolsWindowHelpla6lDaily - Platform - 1 m leftDEV (docker)• жзDOCKERO 81DEV (docker)882APP (-zsh)masterJY-20818-move-AJ-reports-to-separated-datadog-metricJY-20773-fix-automated-reports-user-pilot-trackingJY-20157-AJ-report-not-send-notificationJY-20508-notify-before-AJ-report-expirationJY-20372-ai-reports-promotion-pagesJY-20352-sync-opportunities-without-a-local-owner-user-id-is-nullJY-20738-debug-AJ-tracking-UPJY-18909-automated-reports-ask-jiminnyJY-20692-fix-integration-app-[API_KEY] laysJY-20698-fix-SF-activity-types-on-new-playbookJY-20543-AJ-report-trackingJY-20384-handle-auto-sync-with-no-access-to-event-typeJY-20458-ask-Jiminny-user-definitionsJY-19666-fix-import-contacts-account-associationJY-19666-HS-import-contacts-and-accounts-batch-jobJY-20458-Ask-Jiminny-ReportsJY-20200-batch-update-CRM-objects-SalesforceJY-19666-HS-webhooks-add-contact-and-companyJY-20348-trigger-setup-DI-layout-on-team-creationJY-20326-refactor-info-message-in-commandJY-20317-fix-auto-log-delay-issue-on-all-channels-disabledJY-20312-remove-on-update-change-last-synced-at-crm-configurationsJY-20306-SF-skip-auto-sync-for-task-based-playbookJY-20192-remove-deleted-team-from-saved-search-filtersJY-20197-import-opportunity-batch-jobJY-20293-enable-status-field-for-pipedrive-dealsJY-20191-remove-commands-interactive-promptsJY-20118-change-default-sync-strategyJY-20183-add-cache-on-auto-log-delayJY-20197-add-import-opportunity-batch-job20118-hs-opportunity-make-webhook-strategy-defaultJY-20118-make-default-hs-opportunity-sync-strategy-webhook-basedJY-20196-handle-opportunity-without-noteJY-20118-improve-opportunity-importJY-20189-handle-activity-search-on-deleted-groupsJY-20160JY-20145-filter-out-converted-leads-when-matchingJY-20150-skip-push-summary-on-summary-ready-1f-autologJY-20132-fix-note-encodingJY-19792-clean-logslukas@Lukas-Kovaliks-MacBook-Pro-Jiminny ~/jiminny/app (JY-20725-handle-HS-search-rate-limit) $ devroot@docker_lamp_1:/home/jiminny# ]-zsh84-zsh885100% C8• Mon 11 May 10:04:39181screenpipe™0 ₴6DEV...
|
Slack
|
Jira Cloud (DM) - Jiminny Inc - 3 new items - Slac Jira Cloud (DM) - Jiminny Inc - 3 new items - Slack...
|
NULL
|
15615
|