|
89117
|
console [EU]
SELECT SUM(CASE WHEN o.up…ULL AND s.d console [EU]
SELECT SUM(CASE WHEN o.up…ULL AND s.deleted_at IS NULL… completed.
text/html
text/html
text/html
text/html
text/html
Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89117
|
|
89116
|
console [EU]
SELECT SUM(CASE WHEN o.up…ULL AND s.d console [EU]
SELECT SUM(CASE WHEN o.up…ULL AND s.deleted_at IS NULL… completed.
text/html
text/html
text/html
text/html
text/html
Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
31
3
113
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configur...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89116
|
|
89115
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
31
3
113
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89115
|
|
89114
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
PhpStormFV faVsco.|s ~ViewCooc#12121 on JY-20963-fx-inWindowTO0% L7Thu 28 May 20:53:22ServiceTestCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che© RecordSelector.phg© Activity.phoC) Team.phdcustomuos=laravellogA SF (jiminny@localhostHS_Jocal (jiminny(@localhost)e console (PROD)# composer.isonUmacurayscrict.onaeeeemwwuecorolo.orcomposer.ockdependency-checker.isondewisonids txinkectononoisdMSINSTALL.md20.11.25 KovalikM.lNErNA-wE.h00aiu.?mehuswswo..noocea.cokecontoensyicstocc.mC) PayloadBulider.ohoCoseoweaisto.esseiwice.on© CrmEntitvReoository.ohdcachedelosedbealstagesx5 Cc wtrait UpportunitySynciraitprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stageliminny soneMalicinsre.me13.02.26 KovalkKowakMakerosckaar.lock.son#phpstan.neon.dist#phpstan-baseline.neom«8 ohounit-yml1raw_sqLquery.sgM.README.mesonar-project.properties18.11.25 Kovalik181125 Kovalk18.11.25 Kovalik1R1125 KouaR18.11.25 Kovalik181125 Kouafk181125 Kouan111011181126112111221124Sstage = Sthis->crmentityRepository->getPipelinestageByconditions(crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,2 test.py<> Untitied Diagram.xmlus vetur.config.isif (Sstage === null) ‹sihisoconoreces nultn oscoterosthiso:ClosedDealStages = null:MWSALIOOK GITTEOING IWOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikwe Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot) Stage does not exist => * . Sstageid):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUk0726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.Ws KovalSthis->cachedStages(ScacheKey) = Sstage:T 1olzeditsv Accept Fle x-X Reject File oxg€ 2 of 2 files →return Sstace.A console (EU) Xiid users (EU"A console [STAGINGTEANCoMworeewenneeiealsorce oelcdonktooeooonunim+0.do minnyIn Feb 2025, nomalizeDealStage was still reading probability from the top-level (the old code before the Oct 13 fix). ScSRoy srades031 A9 A31 V3 Y113 Astages created before Oct 1s, 2025 had probability = 8 in the D3. That means getcloscodea (Stages() returned ALL those1783WHERE id = 14269;won stagcs in ost and none in won → every dcal synced -iswon = eBut wait - the oldest updated_at is 2025-02-12, which is before Oct 13, 2025. So those deals were created and last synced1784SELECT COUNT(*)at a time when stages had probability O from the normalization bug. They ve never been re-synced since.Let me verity what percentage were last updated before Oct 13, 2025=470FROM oppontunities cJOIN stages s 1.n<->1: ON s.id = o.stage.idJOIN con configurations cc 1.n<->1: ON cc.id = s.crm.configuration.icWHERE cc.provider = "hubspot"SUM(CASE WHEN o.updated at < '2025-10-13' THEN 1 ELSE @ END) as before_111790AND s-orobability = 106SUM(CASE WHEN o.updated at '2025-10-13* THEN 1 ELSE @ END) as after fix1791ANU ORSAONEOCouNeas Totat—T7Y4ANU OnOeLeTPOROODLRowo060rtunttedo51793AND s.deleted.ars .uLANDo.created.ar >787501806cr_configurations cc ON cc.id = s.crn_configuration_id5792MHERE cc.provider = 'hubspot"E 1796S.orodsouyeAND o.is Mon=&=1797SUMCGASE WHEN o.updated at < *2825-18-13* THEN 1 ELSE 8 END) as before 6ixsan a.deleted at is Milt1798SUMCGASEWHEN o.updated_at >= '2825-10-13' THEN 1 ELSE 0 END) as after_fixoounirx as totall=1800Bur actusliy = herets the renl oicture reaardiess of that soFRON opportunities o— 180%Not stages sune-s: onsde ostagesia1802JOIN crn_configurations cceiweee ceas conf cunarsionRoot Cause (Definitive,normalizedealStaoe before Oct 13, 2025 defaulted probability to e (reading from top-level, not eetadata)). This mean:1RAUініI1.804WHERE cc.provider =hubspotAND S.probability = 188Accept all18051.806AND o.is_won = 8Ack anything (XOL"AND o.deleted_at IS NULLI IIL1807AND s.deleted_at IS NULL:+CodeAdhotrweSarvicesyMbatahaedV AEUiii users 1 s 548 msvA liminny@localhostAHSJlocalAPROD#consoleV ASTAGINGconsoleDockerGtatomontSSLECT SUMICASE WHEN o.undated.atc '2025-10-13' T.ta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xSE EeEre oam es ures where eamildle .5etIW K< 48.009-48,508 v of 48.508Am con provider id mTis HOn TDOLBOBGASTis closed Ystace sd MGuetomid : Istage en oroviden id S482 168887Cien 1hege75482 16888715482 1688871S402 1688071morobabitityIstage undated at m@updated_at V100,00 cnul1:2825-92-12 16:08:8,188,89 <nul1:2825-82-12 15:38:1180,89 <null2825-82-12 15:38:1188.88 coult2025-02-12 15:30:16100.00 cnulE2025-02-12 15:30:09108.00 <null>2025-82-12 15:38:89108.00 <nul1>2025-82-12 15:38:89100.09 <null2825-02-12 15:30:00CSVгллодIcreated. atMrenotely created at Mi teanide2927.11.07. 11.79.202825-82-12 15:38:112024-87-83 89:30-102824-89-84 16:86:822824-87-83 09:38:022825-82-12 15:30:892825-81-31 86:30:842024-07-01 11-30:092021-07-22 14:53:212020-02-25 16:89:482020-01-16 23:88:882929-94-24 15:52-5.2020-09-03 13:45:062020-04-27 14:29:032020-05-15 89:12:592020-10-14 11-27:4315915915915S15Ssortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (17 minutes ago)N Minderd Toome1802.08 UITG.8/enod...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89114
|
|
89113
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
31
3
113
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89113
|
|
89112
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89112
|
|
89111
|
HomeDMsActivityFilesLater..•More+Slack> 0(ah]Fi HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:53:168 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89111
|
|
89110
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89110
|
|
89109
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89109
|
|
89108
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:53:09®8 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89108
|
|
89107
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89107
|
|
89106
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89106
|
|
89105
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
rapstomViewCoocWindowFV faVsco.|s ~#12121 on JY-20963-fx-lCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che# composer.isonCmkcurayocryict.onaeeeemwwuecorolo.or© RecordSelector.phg© Activity.pri;C) Team.phdcomposer.lockdependency-checker.isondewisonhuswswo..noocea.cokecontoensyixstocc.mC) PayloadBulider.ohoCiosooweaistocesserce.on© CrmEntitvReoository.ohdids txcachedelosedbealstagesx5 Cc winkectononoisdMSINSTALL.md20.11.25 Kovaliktrait UpportunitySynciraitM.lNErNA-wE.h00aiu.?meprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stagoliminny sote13.02.26 KovalkMalicinsre.me1110MakerkSstage = Sthis->crmentityRepository->getPipelinestageByconditionsosckaar.lock.son1118#phpstan.neon.dist#phpstan-baseline.neom18.11.25 Kovalik«8 ohounit.yml1R1125 Kovar1raw_sqLquery.sg18.11.25 Kovalik112611211122crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,M.README.me1R1125 KouaRsonar-project.properties18.11.25 Kovalik11242 test.py181125 Kouafk<> Untitied Diagram.xml181125 Kouanus vetur.config.isif (Sstage === null) ‹sihisoiconoroces nult oacoterosthiso:ClosedDealStages = null:MWSALIOOK GIITEOING IVOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikwe Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot) Stage does not exist => * . Sstageid):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUko726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.WS KovalSthis->cachedStages(ScacheKey) = Sstage:T 1ol zeditsv Accept Fle x-X Reject File oxgreturn Sstace.Sarvicesta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xIW K< 48.009-48,508 v of 48.508Am con provider sd mTis HOn TTis closed YDOLBOBGASMstace sd M=custom.log=laravellogA SF jminny@localhostHSJocal (jiminny(@localhost)e console (PROD)A console (EU) Xiii users (EU"A console [STAGINGTeAnddydo minnySELECT osid, o.ern provider id, oxis ron, oxis closed, o,stage e031 49 A29 V 3 У 112 A Vs.con providerid AS stage_con_oroviderid, s.orobability.176315176o.stage_updated_at, o.updated_at, o.created_at, o.remotely_created_at=176=176-1761767=17681769o. teanstFRoh opportuntes oJOIN stages s1.nc->: on s.de o,stageioNot eo confourerions cc1.nc->1: ON cc.id = s.crn_configuration_idWHERE cc.provider = 'hubspotANo suonobabuity =ik1771=177s=17751776F17717751780=170AND o.deleted_at IS NULLnedalatadntsORDER BY o.updated_at DESCselect * fron stages where tean_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296select * fron opportunities where id = 7989974;select * fron stages where id = 14269:SELECT id, crn provider id, name, probability, created at, updated atrkun stadesWHERE 1d= 14269:15178)EizosSELECT COUNT(*)yMbatahaedV AEUiii users 1 s 548 msvA liminny@localhostAHS local: Istage en oroviden id 7:Morobability M S482 168887APROD#consoleV ASTAGINGconsoleDocker Cien 1hege75482 16888715482 1688871S402 1688071Istage undated atm100,00 cnul1:188,89 <nul1:180,89 <null188.88 coult100.00 cnulE108.00 <null>108.00 <nul1>100,09 <nul1)11TO0У L7Thu 28 May 20:53:04ServiceTesteworehsermiehalsorce oelcdonkttoooeooogunifislost- Tnarra/foproperises(eatstool, slosestooe lost;):+0.soats.also - it was updateOrCreate not upsertOpportunity. But more importantly.In Feb 2025, norns Uizedes lStage was still reading probsbility from the too-level (the old code before the Oct 13 fix), Scstages created before Oct 13, 2025 had probability = e in the DB. That means getClosedDea lStages() returned ALL thoseBut wait - the oldest updated at is 2025-02-12, which is before Oct 13, 2025. So those deals were created and last syncecarhee when stades had ororaty olirom the morma fation duo,tthey ve boue deoo thesunoho cincLet me verify what percentage were last updated before Oct 13, 2025SELEGCMICACh HIM Cruodateddt 10clam tiieewniine eonGMIRICE NUCUATed.t'2025-10-13* THEN 1 ELSE @ END) as after fixopportunities oJOIN stages s ON s.id = o.stage 1c30TM em conttourat ions cc OM cosld a swem cont taurationt.Accept allAsk anything (XOL"c Code$ AdaptiveI updated_at Y©created_at Y@renotely_created_at YItean_id y2825-92-12 16:08:8,2927.11.07. 11.79.202021-07-22 14:53:212825-82-12 15:38:12025-02-12 15:30:132025-02-12 15:30:102025-02-12 15:30:092825-82-12 15:38:112824-87-83 09:30:102824-89-84 16:00:042020-02-25 16:89:482020-01-16 23:88:882929-94-24 15:52-5.nanhnanhar do.ta.ar2A0A-00-07 17-46-9%2025-82-12 15:38:892026.02.12 15.79.002020-84-27 14:29:0315S15515S1551592025-82-12 15:38:892825-81-31 86:30:842020-05-15 89:12:592025-02-12 15:30:882024-07-01 11-30:092020-10-14 11-27:4315915Ssortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (17 minutes ago)...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89105
|
|
89104
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:53:048 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89104
|
|
89103
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89103
|
|
89102
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89102
|
|
89101
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
rapstomViewCoocWindowFV faVsco.|s ~#12121 on JY-20963-fx-lCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che# composer.isonMmacuryochyict.onaeeeemwwuecorolo.or© RecordSelector.phg© Activity.pri;C) Team.phdcomposer.lockdependency-checker.isondewisonhuosooswosnoocxea tokecontroensynostorocneC) PayloadBulider.ohoCiosooweaistocesserce.on© CrmEntitvReoository.ohdids txcachedelosedbealstagesx5 Cc winkectononoisdMSINSTALL.md20.11.25 Kovaliktrait UpportunitySynciraitM.lNiEhNALw2.00a1?meprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stagoliminny sote13.02.26 KovalkMalicinsre.me1110MakerSstage = Sthis->crmentityRepository->getPipelinestageByconditionsosckaar.lock.son1118#phpstan.neon.dist#phpstan-baseline.neom18.11.25 Kovalik«8 ohounit.yml1R1125 Kovar1raw_sqLquery.sg18.11.25 Kovalik112611211122crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,M.README.me1R1125 KouaRsonar-project.properties18.11.25 Kovalik11242 test.py181125 Kouafk<> Untitied Diagram.xml181125 Kouanus vetur.config.isif (Sstage === null) ‹sihisoiconoroces nult oacoterosthiso:ClosedDealStages = null:MWSALIOOK GILTEOING IVOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikw Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot Stage does not exist => * . SstageId):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUko726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.WS KovalSthis->cachedStages(ScacheKey) = Sstage:T 1ol zeditsv Accept Fle x-X Reject File oxgreturn Sstace.Sarvicesta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xIW K< 48.009-48,508 v of 48.508Am con provider sd mTis HOn TTis closed YDOLBOBGASMstace sd M=custom.log=laravellogA SF jminny@localhostHS_Jocal (jiminny(@localhost)e console (PROD)A console (EU) Xiii users (EU"A console [STAGINGT AnCo yMeceSELECT osid, o.ern provider id, oxis ron, oxis closed, o,stage e031 A9 A29 M3 Y112 As.con providerid AS stage_con_oroviderid, s.orobability.176115176=176=176-1761767=17681769o. teanstFRoh opportuntes oJOIN stages s1.nc->: on s.de o,stageioNot eo confourerions ccWHERE cc.provider = 'hubspotAND suonobabuty =tee1771AND o.deleted_at IS NULLnedalatadnts=177s=17751776F17717751780=170ORDER BY o.updated at DESCselect * fron stages where team_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296select * fron opportunities where id = 7989974;select * fron stages where id = 14269:rkun stadesWHERE 1d= 14269:15178)EizosSELECT COUNT(*)yMbatahaedV AEUiii users 1 s 548 msvA liminny@localhostAHS local: Istage en oroviden id 7:Morobability M Istage undated atm100,00 cnul1:188,89 <nul1:180,89 <null3519678 1926663919S482 168887APROD#consoleV ASTAGINGconsoleDocker Cien 1hege75482 16888715482 1688871S402 1688071188.88 coult100.00 cnulE108.00 <null>108.00 <nul1>100,09 <nul1)2025-02-12 15:30:102025-02-12 15:30:092025-02-12 15:30:092025-02-12 15:30:692025-02-12 15:30:08ServiceTestalsorce oelcdonkoturooeooonurTO0У L7Thu 28 May 20:52:12+0.2024-09-04 16:00:042024-07-03 09:30:072025-02-12 15:30:092025-01-31 86:30:842024-07-01 11:30:052020-04-24 15:52:562020-09-03 13:45:002020-04-27 14:29:032020-05-15 09:12:592020-10-14 11:27:45155155sortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (16 minutes ago)1760:1 (525 chars, 13 line breaks) UTF-8 4 spac...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89101
|
|
89100
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:52:128 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89100
|
|
89099
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:52:048 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89099
|
|
89098
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89098
|
|
89097
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89097
|
|
89096
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
rapstomFV faVsco.|s ~View#12121 on JY-20963-fix-InCoocWindow€ ServiceTest.php# composer.isoncomposer.lockdependency-checker.isondewisonF ids.txtinkectononoisdMSINSTALL.md20.11.25 KovalkM.lNEhNALwa.00raiu?mejiminny_storageMalicinsre.me13.02.26 KovalikMakerosckaar.lock.sonE phpstan.neon.dist#phpstan-baseline.neom< phpunit.xml1raw_sqLquery.sgM.README.meQ sonar-project.propertiesE test.py<> Untitled Dlagram.xml18.11.25 Kovalik18.11.2518.11.25 Kovallk18.11.25 Kovalik18.11.25 Kovallk18.11.25 Kovalik18.11.25 Kovaik18.11.25 Kovalik18.11.25 Kovalik# vetur.config.isM:WEBHOOK_FILTERING._IMPLEMENTAT18.11.25 Kovalk› Ilb External Librarles18.11.25 Kovalkwe Scratches and consoles18.11.25 Kovalk~ EO Database Consoles18.11.25 KovalkV AEU18.11.25 KovallkA console (EUT18.11.25 KovallkA DEAL RISKS (EU)ko726 KovaldDIEU13.02.26 KovallkA EU TEUN8.WS KovalMmacuryochyict.on© CachedCrmServiceDecorator.php© DeleteObjects Trait.phpRecordSelector.php© MatchActivityCrmData.php© Activity.php© Team,phphuosooswoonoocea.cokecontoensynoscoreoe© PayloadBulider.phpCiosooweaistocesserce.on© CrmÊntity Repository.phpcachedelosedbealstagesx5 Cc wtrait OpportunitySyncTrait1106private function resolveStage(BusinessProcess $businessProcess, ?string Sstageld): ?Stage1116Sstage = Sthis-›crnEntityRepository->getPipelineStageByConditions([CREDIT_CARD]'crn_provider _id' => Sstageld,"type' => Stage::TYPE_OPPORTUNITY,11241f (Sstage === null) (Sintsoiponoredes nultn oacoderosthiso:ClosedDealStages = null;if (Sstage Ra= nuLl) (Sthis->logger->info(*[HubSpot) Stage does not exist => • . $stageId);5050113SSthis-scachedStaces Scachekey = sstage.T 1ol zeditsV Accept Fle x-X Roject Fle oxareturn Sstage:Services+,o, c*v M DatabaseV AEUResult 34@ Result 44xDOLgis_closed Y, QG8A0stage_id YE custom.logE Iaravellog4 SF [iminny@localhost)A HS.Jocal (jiminny@localhost)A console (PROD]A console (EUJxE users (EU)A console (STAGING)T AnCo y68 Jiminny~SELECT O.id, O.crm,providen.id, O.18, XOn, O.18,closed, D. stage &.,031 49 A29 X3 X112 A V1761s.crn_provider_id AS stage_crn_provider_id, s.probability.15176o.stage_updated_at, o.updated_at, o.created_at, o.renotely_created_at,-1763o.tean_id=1764FRoh opportuntes o=1765=1766JOIN stages s1.n<->1: ON s.id = o.stage_idNot eo confourerions cc17671.n<->1: ON cc.id = s.crm_configuration_id=1768WHERE CC.provider = 'hubspes"1769AND S.probability = 100177017711772=177sAND o.deleted_at IS NULLAND s.deleted_at IS NULLORDER BY o.updated_at DESC;=17751776select * fron stages where tean_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296F177-17791780select * fron opportunities where id = 7989974;select * fron stages where id = 14269;=170-178471783rkun stadesWHERE id = 14269;1784EizosSELECT COUNT(*)ti users 1 s 548 msv 4/jiminny@localhost4 HSJJocalV A PROD#consoleV A STAGINGconsole& Dockerta outoutul liminnv coportunitiesisstiminny stadeIW K< 48,009-48,508 v of 48,508g crn_provider_id Y1500374 S Tis HOn T@stage_crn_provider_id T @probability Y@stage_updated_at V100.00 <null>100.00 <nuLl>100.00 <nuL2>100.00 <nu22>100.00 <nul1>100.00 <nul1>100.00 <null>100.00 <nul2>2025-02-12 15:30:892025-02-12 15:30:092025-02-12 15:30:692025-02-12 15:30:08CascadeTlasorce DaleionkServiceTestOtwooeoo0nturTO0У L7inu co moy kuiotros+0.Accopt all• 02824-87-83 89:30:072025-02-12 15:30:092025-01-31 86:30:842024-07-01 11:30:052020-09-03 13:45:002020-04-27 14:29:032020-05-15 09:12:592020-10-14 11:27:45ortcuts conflicts: Clone Caret Below and 1 more shortcut conflict with macOS shortcuts. Modity these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (16 minutes ago)15515515s1551760:1 (525 chars, 13 line breaks) UTF-8 4 spac...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89096
|
|
89095
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:51:33®8 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89095
|
|
89094
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:51:258 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89094
|
|
89093
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89093
|
|
89092
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:51:178 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89092
|
|
89091
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89091
|
|
89090
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:51:09®8 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89090
|
|
89089
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89089
|
|
89088
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
rapstomViewCoocWindowFV faVsco.|s ~#12121 on JY-20963-fx-lCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che# composer./sonCmkcurayocryict.onaeeeemwwuecorolo.or© RecordSelector.phg© Activity.pri;C) Team.phdcomposer.lockdependency-checker.isondewisonhuswswo..noocea.cokecontoensyixstocc.mC) PayloadBulider.ohoCiosooweaistocesserce.on© CrmEntitvReoository.ohdids txcachedelosedbealstaaesx5 Cc winkectononoisdMSINSTALL.md20.11.25 Kovaliktrait UpportunitySynciraitM.lNErNA-wE.h00aiu.?meprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stagoliminny sote13.02.26 KovalkMalicinsre.me1110MakerSstage = Sthis->crmentityRepository->getPipelinestageByconditionsosckaar.lock.son1118#phpstan.neon.dist#phpstan-baseline.neom18.11.25 Kovalik«8 ohounit.yml1R1125 Kovar1raw_sqLquery.sg18.11.25 Kovalik112611211122crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,M.README.me1R1125 KouaRsonar-project.properties18.11.25 Kovalik11242 test.py181125 Kouafk<> Untitied Diagram.xml181125 Kouanus vetur.config.isif (Sstage === null) ‹sihisoiconoroces nult oacoterosthiso:ClosedDealStages = null:MWSALIOOK GIITEOING IVOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikw Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot) Stage does not exist => * . Sstageid):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUko726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.WS KovalSthis-scachedStacesScachekey = Sstage:T 1olzeditsv Accept Fle x-X Reject File oxgreturn Sstace.Sarvicesta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xIW K< 48.009-48,508 v of 48.508Am con provider sd mTis HOn TTis closed YDOLBORRASMstace sd M=custom.log=laravellogA SF jminny@localhostHS_Jocal (jiminny(@localhost)e console (PROD)A console (EU) Xiii users (EU"A console [STAGINGTeAnddydo minnySELECT osid, o.ern provider id, oxis ron, oxis closed, o,stage e031 49 A29 V 3 У 112 A Vs.con providerid AS stage_con_oroviderid, s.orobability.176315176o.stage_updated_at, o.updated_at, o.created_at, o.remotely_created_at=176=176-1761767=17681769o. teanstFRoh opportuntes oJOIN stages s.ne-y: oN sod e oustagesialNot eor conflouratsions ce1.nc->1: ON cc.id = s.crn_configuration_idWHERE cc.provider = 'hubspotANo suonobabuity =ik1771=177s=17751776F17717751780=170AND o.deleted_at IS NULLnedalatadntsORDER BY o.updated at DESCselect * fron stages where tean_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296select * fron opportunities where id = 7989974;select * fron stages where id = 14269:SELECT id, crn provider id, name, probability, created at, updated atrkun stadesWHERE 1d= 14269:15178)EizosSELECT COUNT(*)yMbatahaedV AEUGii users 1 s 548 msvA liminny@localhostAHS local: Istage en oroviden id 7:Morobability M S482 168887APROD#consoleV ASTAGINGconsoleDocker Cien 1hege75482 16888715482 1688871S402 1688071Istage undated atm100,00 cnul1:188,89 <nul1:180,89 <null188.88 coult100.00 cnulE108.00 <null>108.00 <nul1>100,09 <nul1)mtnnmnTO0У L7ServiceTestworeewennetiealsorce oelcdonkoturooeooonurThe oldest updated_at = 2025-02-12 is the key clue. Let me check what happened around that date;pp/Services/Cra/Hubspot/Service-php s/o/Reposttorses/tra/ErPEntityRepository-phfo git log -oneline -afters"2025-01-15" -before= 2825-82-1551fle +2 >Ack anything (XOL+ «Code$ AdaptiveIupdated_at Y2825-82-12 16:00:8.2025-02-12 15:30:112025-02-12 15:30:112025-02-12 15:30:102025-02-12 15:30:092025-02-12 15:30:092025-82-12 15:38:892025-02-12 15:30:88Icreated.at Y2823-11-83 11:38:292825-82-12 15:30:112824-87-83 09:30:102824-89-84 16:00:042024-87-83 09:30:072825-82-12 15:30:892825-81-31 86:30:842024-07-01 11-30:09@renotely_created_at Y2021-07-22 14:53:212020-02-25 16:89:482020-01-16 23:00:002020-84-24 15:52:562020-09-03 13:45:002020-84-27 14:29:032020-05-15 89:12:592020-10-14 11-27:43I tean_id YCSV15S15515515S15915915Ssortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (15 minutes ago)Thu 28 May 20:50:56+0.Pun su CkinAccept all...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89088
|
|
89087
|
Project: faVsco.js, menu
HomeDMsActivityFilesLater Project: faVsco.js, menu
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:568 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89087
|
|
89086
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89086
|
|
89085
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89085
|
|
89084
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
rapstomViewCoocWindowFV faVsco.|s ~#12121 on JY-20963-fx-lCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che# composer.isonMmacuryochyict.onaeeeemwwuecorolo.or© RecordSelector.phg© Activity.pri;C) Team.phdcomposer.lockdependency-checker.isondewisonhuswswo..noocea.cokecontoensyixstocc.mC) PayloadBulider.ohoCoseweaisto.esseiwice.on© CrmEntitvReoository.ohdids txcachedelosedbealstaaesx5 Cc winkectononoisdMSINSTALL.md20.11.25 Kovaliktrait UpportunitySynciraitM.lNErNA-wE.h00aiu.?meprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stagoliminny sote13.02.26 KovalkMalicinsre.me1110MakerkSstage = Sthis->crmentityRepository->getPipelinestageByconditionsosckaar.lock.son1118#phpstan.neon.dist#phpstan-baseline.neom18.11.25 Kovalik«8 ohounit-yml1R1125 Kovar1raw_sqLquery.sg18.11.25 Kovalik112611211122crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,M.README.me1R1125 KouaRsonar-project.properties18.11.25 Kovalik11242 test.py181125 Kouafk<> Untitied Diagram.xml181125 Kouanus vetur.config.isif (Sstage === null) ‹sintsoinconoreces nult sacoterosthiso:ClosedDealStages = null:MWSALIOOK GIITEOING IVOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikw Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot) Stage does not exist => * . Sstageid):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUko726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.WS KovalSthis-scachecStaces.scachekey = Sstade.T 1ol zeditsv Accept Fle x-X Reject File oxgreturn Sstace.Sarvicesta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xIW K< 48.009-48,508 v of 48.508Am con provider sd mTis HOn TTis closed YDOLBORRASMstace sd M=custom.log=laravellogA SF jminny@localhostHS_Jocal (jiminny(@localhost)e console (PROD)A console (EU) Xiii users (EU"A console [STAGINGT AnCo ydo minnySELECT osid, o.ern provider id, oxis ron, oxis closed, o,stage e031 A9 A29 M3 Y112 As.con providerid AS stage.con_oroviderid, s,orobability.176315176o.stage_updated_at, o.updated_at, o.created_at, o.renotely_created_at=176=176-1761767=17681769o. teanstFRoh opportuntes oJOIN stages s.ne-y: oN sod e oustagesialNot eor conflouratsions ce1.nc->1: ON cc.id = s.crn_configuration_idWHERE cc.provider = 'hubspotANo suonobabuity =ik1771AND o.deleted_at IS NULLnedeltarntsi=177s=17751776F17717751780=170ORDER BY o.updated at DESCselect * fron stages where tean_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296select * fron opportunities where id = 7989974;select * fron stages where id = 14269:SELECT id, crn provider id, name, probability, created at, updated atrkun stadesWHERE 1d= 14269:15178)EizosSELECT COUNT(*)yMbatahaedV AEUGii users 1 s 548 msvA liminny@localhostAHS local: Istage en oroviden id 7:Morobability M Istage undated atm100,00 cnul1:188,89 <nul1:180,89 <null3519678 1926663919S482 168887APROD#consoleV ASTAGINGconsoleDocker Cien 1hege75482 16888715482 1688871S402 1688071188.88 coult100.00 cnulE108.00 <null>108.00 <nul1>100,09 <nul1)TO0У L7Thu 28 May 20:50:44ServiceTestalsorce oelcdonkoturooeooonur+0.Coun(*) 39234, Also I noticed that vo.stage_updated_at, o.updated_at, o.created_at, o.remotely_created_ain SELECT o.id, o.crm_provider_jd, o.is_won, o.is_closed, o.stage_iceise is causing this at scaltThe oldest updated at = 2025-02-12 is the key clue. Let me check what happened around that datesRunxw SkipAccept all• 0Ask anything (XOL)• <> CodelAdhotrweI updated_at Y2825-92-12 16:08:8,2025-02-12 15:30:112025-02-12 15:30:112025-02-12 15:30:102025-02-12 15:30:092025-02-12 15:30:092025-82-12 15:38:892025-02-12 15:30:88Dereted.at Y2823-11-83 11:38:292825-82-12 15:30:112824-87-83 09:30:102824-89-84 16:00:042024-07-03 09:30:072825-82-12 15:30:892025-01-31 86:30:842024-07-01 11:30:05:I renotely_created_at V2021-07-22 14:53:22020-02-25 16:09:482020-01-16 23:00:002020-04-24 15:52:562020-09-03 13:45:002020-84-27 14:29:032020-05-15 09:12:592020-10-14 11-27:43sortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (15 minutes ago)csv~IQ tean,id T15S15515515S15S15S15515S12804 /605 chare 18 tho hroskelUTG pAend...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89084
|
|
89083
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:448 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89083
|
|
89082
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89082
|
|
89081
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
rapstomViewCoocWindowFV faVsco.|s ~#12121 on JY-20963-fx-lCSamviceTesconrDeleteObjectsTrait.phgCMatchiet witermonts che# composer.isonCmkcurayocryict.onaeeeemwwuecorolo.or© RecordSelector.phg© Activity.pri;C) Team.phdcomposer.lockdependency-checker.isondewisonhusoswo..noocea.tokecontroeusynostode.C) PayloadBulider.ohoCoseweaisto.esseiwice.on© CrmEntitvReoository.ohdids txcachedelosedbealstagesx5 Cc winkectononoisdMSINSTALL.md20.11.25 Kovaliktrait UpportunitySynciraitM.lNiEhNALw2.00a1?meprivate function resolveStage(BusinessProcess SbusinessProcess, ?string Sstageld): ?Stagoliminny sote13.02.26 KovalkMalicinsre.me1110MakerSstage = Sthis->crmentityRepository->getPipelinestageByconditionsosckaar.lock.son1118#phpstan.neon.dist#phpstan-baseline.neom18.11.25 Kovalik«8 ohounit-yml1R1125 Kovar1raw_sqLquery.sg18.11.25 Kovalik112611211122crn_provider_1d => Sstageld'type' => Stage::TYPE_OPPORTUNITY,M.README.me1R1125 KouaRsonar-project.properties18.11.25 Kovalik11242 test.py181125 Kouafk<> Untitied Diagram.xml181125 Kouanus vetur.config.isif (Sstage === null) ‹sihisoconoreces nultn oscoterosthiso:ClosedDealStages = null:MWSALIOOK GITTEOING IWOIGUISUTAT18.11.25 Kovaliik› (h External Libraries18.11.25 Kovalikwe Scratches and consoles18.11.25 Kovalikv Database Consoles18.11.25 Kovalkif (Sstage === null)Sthis->logger->info(*(HubSpot) Stage does not exist => * . Sstageid):V AEU18.11.25 KovaltA console (EUT18.11.25 KovaltA DEAL RISKS (EUko726 KovaldDIEU13.02.26 Kovalt50501139A EU TEUT8.WS KovalSthis-scachedStaces Scachekey = Sstage:T 1ol zeditsv Accept Fle x-X Reject File oxgreturn Sstace.sarvicesta outoutul liminnv coportunitiesisstiminny stade18 Recult 34il Result 44 xIW K< 48.009-48,508 v of 48.508Am con provider sd mTis HOn TTis closed YDOLBORRASMstace sd M=custom.log=laravellogA SF jminny@localhostHSJocal (jiminny(@localhost)e console (PROD)A console (EU) Xiii users (EU"A console [STAGINGT AnCo ydo minnySELECT osid, o.ern provider id, oxis ron, oxis closed, o,stage e031 A9 A29 V3 Y 112 As.con providersid AS stage_con_oroviderid, s,orobability.176315176o.stage_updated_at, o.updated_at, o.created_at, o.renotely_created_at=176=176-1761767=17681769o.tean.FRoh opportuntes oJOIN stages s1.nc->: on s.de o,stageioNot eo confourerions cc1.nc->1: ON cc.id = s.crn_configuration_idWHERE cc.provider = 'hubspotANo suonobabuity =ik1771AND o.deleted_at IS NULLnedeltarntsi=177s=17751776F17717751780=170ORDER BY o.updated_at DESCselect * fron stages where tean_id = 416;# 14267, 14269, 14283, 14293, 14294, 14296select * fron opportunities where id = 7989974;select * fron stages where id = 14269:SELECT id, crn provider id, name, probability, created at, updated atrkun stadesWHERE 1d= 14269:15178)EizosSELECT COUNT(*)yMbatahaedV AEUiii users 1 s 548 msvA liminny@localhostAHS local: Istage en oroviden id 7:Morobability M Istage undated atm100,00 cnul1:188,89 <nul1:180,89 <null3519678 1926663919S482 168887APROD#consoleV ASTAGINGconsoleDocker Cien 1hege75482 16888715482 1688871S402 1688071188.88 coult100.00 cnulE108.00 <null>108.00 <nul1>100,09 <nul1)TO0У L7Thu 28 May 20:50:30ServiceTestCascadeeworehsermiehalsorce oelcdonktooreooonuniAND S-probability = 100AND o.is won = eRoo.oclercoras NULAND Shdelered ateis NuLIAND o.created_at > *2025-10-13';+0.ilthe count drops dramatically, the bulk is historical. The few post-Oct-13 ones (including today's deal) are Ekely from thestale cache buo voutve already "ixad =a batch that handened to triader (coortStaops miienunorobablty 100 = out that's imoossible since it's been 100 tor 5 monthsUnless this deal was created in a batch that also had another team's config being processed - but that can't happen sinceThe most likely explanation for the today deal: it was caught in the same batch as another deal that triggered importStagesoramiscna staaa), which ma-noonlated cachedelosadhan Sthoas befoze feoortSthoes tan - eyactly tha bua vonie tixedEven though stage 14269 already had probability 100, if cachedClosedbealStages was populated before importStages ranDACCho AllnCount(*) 39234. Also I noticed that when i run SELECT o.id, o.crm provider id, o.is won, o.is closed, o.stage icI updated_at Y2825-92-12 16:08:8,2025-02-12 15:30:112025-02-12 15:30:112025-02-12 15:30:102025-02-12 15:30:692025-02-12 15:30:092025-82-12 15:38:892025-02-12 15:30:88@created_at V2823-11-83 11:38:292825-82-12 15:38:112824-87-83 09:30:102824-89-84 16:00:04@renotely_created_at Y2021-07-22 14:53:212020-02-25 16:89:482020-01-16 23:00:002020-84-24 15:52:562825-82-12 15:30:892024-07-01 11-30:092020-84-27 14:29:032020-05-15 89:12:592020-10-14 11-27:43B tean_id yCSV15S155155155155sortcuts conflicts: Clone Caret Below and 1 more shortcut confiict with macOS shortcuts. Modify these shortcuts or change macOS system settings. // Modity Shortcuts // Don't Show Again (15 minutes ago)...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89081
|
|
89080
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:288 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89080
|
|
89079
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89079
|
|
89078
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89078
|
|
89077
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:228 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89077
|
|
89076
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89076
|
|
89075
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89075
|
|
89074
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89074
|
|
89073
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:06®8 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89073
|
|
89072
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
HomeDMsActivityFilesLater..•More+Slack> 0(ah]FileEditViewGoHistoryJiminny ...* Starredplatform-backend-...platform-inner-teamChannels# ai-chapter# alerts# backend# bugs# confusion-clinic# donut_time# engineering# general# happy_birthday& infosec_internal_all# infra-changes# infrastructure_dev# jbu-team-info# jiminny-bg# platform-team# platform-tickets# product_launches# random# releases# sofia-office# support# thank-yous# the_people_of_jimi...WindowHelpSearch: in:#platform-inner-teamX*à platform-inner-teamMessagesBookmarks0 Channel Overview7 RefinementsO Files< PinsP Retro Action ItemsChanges:Monday, May 4th ~• Do not requeue duplicatesjiminny/app May 4th Added by GitHubNikolay Ivanov 1:48 PMhttps://github.com/jiminny/app/pull/12041, фикс за импорта на стейджове (само при hubspot се оказа )но има още една грешка за тях, слагаме стейджове от една организация на другаи не мога да намеря никьде в кода защона еи има две организации които не могат да се изтрият заради товаVasil Vasilev 2:07 PMимаше проблем със мачването на стейджове от една организация на другазаради кеширане на pipelineldпреди горе долу 10тина дена го гледахмеNikolay Ivanov 2:07 PMтой още го иматози пьт при stageвиж ми PRVasil Vasilev 2:08 PMpipelineld бeшe default за едно 5-6 организации, и там се омазваше кешапонеже няма нито crm coniguration id, нито team id включено в кеша дето заминава в РЕДИСа, да, това е сьщотодобре де, аз защо си мисля, че фикс за тоя проблем вече замина на продNikolay Ivanov 2:10 PMMessage & platform-inner-team+100% <78 • Thu 28 May 20:50:048 10Untitled +...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89072
|
|
89071
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89071
|
|
89070
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89070
|
|
89069
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89069
|
|
89068
|
Project: faVsco.js, menu
#12121 on JY-20963-fix-im Project: faVsco.js, menu
#12121 on JY-20963-fix-import-on-deleted-entity, menu
Start Listening for PHP Debug Connections
ServiceTest
Run 'ServiceTest'
Debug 'ServiceTest'
More Actions
JetBrains AI
Search Everywhere
IDE and Project Settings
Show Replace Field
Search History
cachedClosedDealStages
New Line
Match Case
Words
Regex
Replace History
Replace
New Line
Preserve case
6/6
Previous Occurrence
Next Occurrence
Filter Search Results
Open in Window, Multiple Cursors
Click to highlight
Close
Sync Changes
Hide This Notification
Code changed:
Hide
75
2
22
Previous Highlighted Error
Next Highlighted Error
<?php
declare(strict_types=1);
namespace Jiminny\Services\Crm\Hubspot\ServiceTraits;
use Carbon\Carbon;
use HubSpot\Client\Crm\Deals\Model\CollectionResponseAssociatedId;
use Jiminny\Exceptions\InvalidArgumentException;
use Jiminny\Models\Account;
use Exception;
use Jiminny\Component\DealInsights\Forecast\Forecast;
use Jiminny\Jobs\Crm\MatchActivitiesToNewOpportunity;
use Jiminny\Models\Contact;
use Jiminny\Models\Crm\BusinessProcess;
use Jiminny\Exceptions\CrmException;
use Jiminny\Models\Crm\Profile;
use Jiminny\Models\Opportunity;
use Illuminate\Support\Collection;
use Jiminny\Models\Stage;
use Jiminny\Repositories\Crm\CrmEntityRepository;
use Jiminny\Services\Crm\Hubspot\DealFieldsService;
use Jiminny\Services\Crm\Hubspot\OpportunitySyncStrategy\HubspotSingleSyncStrategy;
use Jiminny\Services\Crm\Hubspot\WebhookSyncBatchProcessor;
use Jiminny\Services\Crm\OpportunitySyncStrategyResolver;
use Jiminny\Utils\CurrencyFormatter;
/**
* Optimized sync methods for better performance
* These methods can be integrated into SyncCrmEntitiesTrait for significant performance gains
*/
trait OpportunitySyncTrait
{
private const int BATCH_SIZE = 100;
private const int BATCH_PROCESS_SIZE = 800;
protected OpportunitySyncStrategyResolver $opportunitySyncStrategyResolver;
protected CrmEntityRepository $crmEntityRepository;
protected DealFieldsService $dealFieldsService;
private ?array $cachedClosedDealStages = null;
private array $cachedBusinessProcesses = [];
private array $cachedStages = [];
/** @var array<string, array<string>> keyed by config id */
private array $cachedOpportunitySyncableFields = [];
/** @var array<string, mixed> keyed by configId:ownerId */
private array $cachedOwnerProfiles = [];
/** @var array<string, mixed> keyed by configId:businessProcessId */
private array $cachedRecordTypes = [];
public function syncOpportunities(array $parameters, ?string $strategy = null): int
{
$startTime = microtime(true);
$strategies = $this->opportunitySyncStrategyResolver->getStrategies($this->config, $strategy);
$parameters['config'] = $this->config;
$syncCount = 0;
$reportedTotal = 0;
$lastSyncedId = [];
$strategyNames = [];
try {
foreach ($strategies as $strategyName => $syncStrategy) {
$strategyNames[] = $strategyName;
$this->logger->info(
'[' . $this->getDisplayName() . '] Syncing opportunities using strategy: ' . $strategyName,
['team' => $this->team->getId()]
);
$total = 0;
$lastId = null;
$buffer = [];
// HubspotWebhookBatchSyncStrategy returns empty generator, this is for other strategies
foreach ($syncStrategy->fetchOpportunities($parameters, $total, $lastId) as $hsOpportunity) {
$buffer[] = $hsOpportunity;
// process every 800 rows (fits < 1 000 association limit)
if (\count($buffer) >= self::BATCH_PROCESS_SIZE) {
$syncCount += $this->processOpportunityBatch($buffer);
$buffer = [];
}
}
// leftovers
if ($buffer) {
$syncCount += $this->processOpportunityBatch($buffer);
}
$reportedTotal += $total;
$lastSyncedId = $lastId;
}
} catch (\HubSpot\Client\Crm\Deals\ApiException | CrmException $e) {
$this->handleSyncException($e, $parameters);
}
$durationMs = round((microtime(true) - $startTime) * 1000, 2);
$this->logger->info(
'[HubSpot] Synced opportunities',
[
'team' => $this->team->getId(),
'strategies' => implode(',', $strategyNames),
'sync_count' => $syncCount,
'total' => $reportedTotal,
'last_synced_id' => $lastSyncedId,
'duration_ms' => $durationMs,
]
);
return $reportedTotal;
}
private function handleSyncException(\Throwable $e, array $parameters): void
{
if (($parameters['since'] ?? null) instanceof Carbon) {
$parameters['since'] = $parameters['since']->toDateTimeString();
}
$parameters['config'] = $this->config->getId();
$this->logger->warning('[' . $this->getDisplayName() . '] Sync opportunities failed', [
'teamId' => $this->team->getUuid(),
'parameters' => $parameters,
'reason' => $e->getMessage(),
]);
}
/**
* @inheritdoc
*/
public function syncOpportunity(string $crmId): ?Opportunity
{
$strategy = $this->opportunitySyncStrategyResolver->resolve(
$this->config,
OpportunitySyncStrategyResolver::SINGLE_SYNC_OPPORTUNITY_STRATEGY,
);
$parameters = [
'config' => $this->config,
'crm_id' => $crmId,
];
try {
if (! $strategy instanceof HubspotSingleSyncStrategy) {
throw new InvalidArgumentException('Strategy must by HubspotSingleSyncStrategy');
}
$hsOpportunity = $strategy->fetchOpportunity($parameters);
} catch (\HubSpot\Client\Crm\Deals\ApiException $e) {
$this->logger->info('[' . $this->getDisplayName() . '] Opportunity not found', [
'teamId' => $this->team->getUuid(),
'crmId' => $crmId,
'reason' => $e->getMessage(),
]);
return null;
}
$hsOpportunity['associations'] = $this->convertDealAssociations($hsOpportunity['associations'] ?? []);
return $this->importOrUpdateOpportunity($hsOpportunity);
}
/**
* Process webhook-collected opportunity batches.
*
* Drains Redis sets containing company CRM IDs collected from webhook events
* and dispatches ImportOpportunityBatch jobs for batch processing.
*
* @return int Number of opportunity IDs dispatched to jobs
*/
public function batchSyncOpportunities(): int
{
$configId = $this->team->getCrmConfiguration()->getId();
return $this->batchProcessor->processBatchesForObjectType(
WebhookSyncBatchProcessor::OBJECT_TYPE_DEAL,
$configId
);
}
/**
* Import a batch of opportunities by their CRM IDs.
* Fetches opportunity data from HubSpot API and delegates to importOpportunityBatch().
*
* @param array<string> $crmIds HubSpot deal CRM IDs
*
* @return array{success: array, failed_ids: array, errors?: array<string, string>}
*/
public function importOpportunityBatchByIds(array $crmIds): array
{
$fields = $this->dealFieldsService->getFieldsForConfiguration($this->config);
$allDeals = [];
foreach (array_chunk($crmIds, self::BATCH_SIZE) as $chunk) {
$deals = $this->client->getOpportunitiesByIds($chunk, $fields);
foreach ($deals as $deal) {
$allDeals[] = $deal;
}
}
// IDs not returned by HubSpot are likely deleted or inaccessible deals.
// These are not failures — retrying won't bring them back.
$fetchedIds = array_map('strval', array_column($allDeals, 'id'));
$notFoundIds = array_values(array_diff(array_map('strval', $crmIds), $fetchedIds));
if (! empty($notFoundIds)) {
$this->logger->info('[' . $this->getDisplayName() . '] CRM IDs not found in HubSpot (likely deleted)', [
'teamId' => $this->team->getId(),
'notFoundCount' => \count($notFoundIds),
'notFoundIds' => $notFoundIds,
'requestedCount' => \count($crmIds),
'fetchedCount' => \count($allDeals),
]);
}
if (empty($allDeals)) {
return ['success' => [], 'failed_ids' => []];
}
return $this->importOpportunityBatch($allDeals);
}
private function getClosedDealStages(): array
{
if ($this->cachedClosedDealStages !== null) {
return $this->cachedClosedDealStages;
}
$stages = $this->crmEntityRepository->getOpportunityClosedStages($this->config);
$data = [
'lost' => [],
'won' => [],
];
foreach ($stages as $stage) {
if ($stage->probability == 0.00) {
$data['lost'][] = $stage->crm_provider_id;
}
if ($stage->probability == 100.00) {
$data['won'][] = $stage->crm_provider_id;
}
}
$this->cachedClosedDealStages = $data;
return $data;
}
/**
* Import deals into the database with pre-fetched associations.
*
* API calls here (getAssociationsData, getExistingOpportunityCrmIds) are NOT
* caught — if they throw, the exception propagates to ImportOpportunityBatch::handle()
* where Laravel retries the whole job with backoff. After all retries exhausted,
* failed() requeues all IDs to Redis.
*
* The per-deal loop catches exceptions individually. A deal can end up in three states:
* - success: imported/updated successfully
* - failed_ids: exception thrown (DB constraint violation, corrupt data, etc.)
* These are permanent issues — retrying won't fix them.
* - skipped (null): missing dependencies (no account, unknown pipeline/stage).
* This is acceptable — the deal cannot be imported until those exist.
*/
private function importOpportunityBatch(array $deals): array
{
$syncedOpportunities = [
'success' => [],
'failed_ids' => [],
];
$dealIds = array_column($deals, 'id');
$batchStart = microtime(true);
$slowDeals = [];
// Shared association/existing-ID preparation is batch-level state. If it fails, rethrow so the
// queue job retries the whole batch and eventually requeues all deal IDs back to Redis.
try {
$companyAssocStart = microtime(true);
$companyAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'companies');
$companyAssocMs = (int) round((microtime(true) - $companyAssocStart) * 1000);
$contactAssocStart = microtime(true);
$contactAssociations = $this->client->getAssociationsData($dealIds, 'deals', 'contacts');
$contactAssocMs = (int) round((microtime(true) - $contactAssocStart) * 1000);
$prepareStart = microtime(true);
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
$prepareTimings = [];
$associationsData = $this->prepareAssociatedEntities(
$companyAssociations,
$contactAssociations,
$prepareTimings
);
$prepareMs = (int) round((microtime(true) - $prepareStart) * 1000);
$missingCompanies = count(array_diff(
$allCompanyIds,
array_keys($associationsData['company_id_mappings'] ?? [])
));
$missingContacts = count(array_diff(
$allContactIds,
array_keys($associationsData['contact_id_mappings'] ?? [])
));
$existingCrmIds = $this->crmEntityRepository->getExistingOpportunityCrmIds(
$this->config,
array_map('strval', $dealIds)
);
$existingCrmIdSet = array_flip($existingCrmIds);
} catch (\Throwable $e) {
$this->logger->error('[' . $this->getDisplayName() . '] Failed to fetch associations or existing IDs', [
'teamId' => $this->team->getId(),
'dealCount' => count($dealIds),
'error' => $e->getMessage(),
]);
throw $e;
}
$loopStart = microtime(true);
foreach ($deals as $deal) {
$dealStart = microtime(true);
try {
$deal['associations'] = $this->prepareAssociationsForOpportunity(
$deal['id'],
$companyAssociations,
$contactAssociations,
$associationsData
);
$syncedOpportunity = $this->importOrUpdateOpportunity(
$deal,
isset($existingCrmIdSet[(string) $deal['id']])
);
if ($syncedOpportunity) {
$syncedOpportunities['success'][] = $syncedOpportunity;
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import opportunity', [
'teamId' => $this->team->getId(),
'crmId' => $deal['id'],
'error' => $e->getMessage(),
]);
$syncedOpportunities['failed_ids'][] = $deal['id'];
$syncedOpportunities['errors'][$deal['id']] = $e->getMessage();
}
$dealMs = (int) round((microtime(true) - $dealStart) * 1000);
if ($dealMs > 1000) {
$slowDeals[] = ['crmId' => $deal['id'], 'ms' => $dealMs];
}
}
$loopMs = (int) round((microtime(true) - $loopStart) * 1000);
$totalMs = (int) round((microtime(true) - $batchStart) * 1000);
$this->logger->info('[' . $this->getDisplayName() . '] importOpportunityBatch timing', [
'teamId' => $this->team->getId(),
'deal_count' => count($deals),
'total_ms' => $totalMs,
'company_assoc_api_ms' => $companyAssocMs,
'contact_assoc_api_ms' => $contactAssocMs,
'prepare_entities_ms' => $prepareMs,
'prepare_accounts_ms' => $prepareTimings['accounts_ms'],
'prepare_contacts_ms' => $prepareTimings['contacts_ms'],
'total_companies' => count($allCompanyIds),
'missing_companies' => $missingCompanies,
'total_contacts' => count($allContactIds),
'missing_contacts' => $missingContacts,
'deals_loop_ms' => $loopMs,
'avg_deal_ms' => ! empty($deals) ? (int) round($loopMs / count($deals)) : 0,
'slow_deals_count' => count($slowDeals),
'slow_deals' => array_slice($slowDeals, 0, 10),
]);
return $syncedOpportunities;
}
/**
* Prepare associated entities for opportunities with optimized batch processing
* Returns structured data with CRM ID to DB ID mappings for each opportunity
*/
private function prepareAssociatedEntities(
array $companyAssociations,
array $contactAssociations,
array &$timings = []
): array {
// Step 1: Collect all unique company and contact IDs from associations
$allCompanyIds = $this->flattenAssociationIds($companyAssociations);
$allContactIds = $this->flattenAssociationIds($contactAssociations);
// Step 2: Batch sync missing entities and get CRM ID to DB ID mappings
$companyIdMappings = [];
$contactIdMappings = [];
$accountsMs = 0;
$contactsMs = 0;
if (! empty($allCompanyIds)) {
$start = microtime(true);
$companyIdMappings = $this->prepareAssociatedAccounts($allCompanyIds);
$accountsMs = (int) round((microtime(true) - $start) * 1000);
}
if (! empty($allContactIds)) {
$start = microtime(true);
$contactIdMappings = $this->prepareAssociatedContacts($allContactIds);
$contactsMs = (int) round((microtime(true) - $start) * 1000);
}
$timings = [
'accounts_ms' => $accountsMs,
'contacts_ms' => $contactsMs,
];
return [
'company_id_mappings' => $companyIdMappings,
'contact_id_mappings' => $contactIdMappings,
];
}
/**
* Flatten association data to get unique IDs
*/
private function flattenAssociationIds(array $associations): array
{
$ids = [];
foreach ($associations as $dealAssociations) {
if (is_array($dealAssociations)) {
foreach ($dealAssociations as $id) {
$ids[$id] = true;
}
}
}
return array_keys($ids);
}
/**
* Batch sync missing accounts
*/
private function prepareAssociatedAccounts(array $companyIds): array
{
// Find which accounts already exist (lean covering-index lookup)
$existingAccountsData = $this->crmEntityRepository
->getExistingAccountIdsMap($this->config, $companyIds);
$missingCompanyIds = array_diff($companyIds, array_keys($existingAccountsData));
if (empty($missingCompanyIds)) {
return $existingAccountsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing accounts', [
'teamId' => $this->team->getUuid(),
'total_companies' => count($companyIds),
'existing_companies' => count($existingAccountsData),
'missing_companies' => count($missingCompanyIds),
]);
// we already have limit on opportunity ids count
// Initialize variable before try block
$syncedAccountsData = [];
try {
$syncedAccountsData = $this->batchSyncCrmObjects('companies', $missingCompanyIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing accounts', [
'size' => count($missingCompanyIds),
'error' => $e->getMessage(),
]);
$syncedAccountsData = [];
}
return $existingAccountsData + $syncedAccountsData;
}
/**
* Prepare associated contacts - find existing and sync missing ones
* Returns mapping of CRM ID to DB ID
*/
private function prepareAssociatedContacts(array $contactIds): array
{
// Find which contacts already exist (lean covering-index lookup)
$existingContactsData = $this->crmEntityRepository
->getExistingContactIdsMap($this->config, $contactIds);
$missingContactIds = array_diff($contactIds, array_keys($existingContactsData));
if (empty($missingContactIds)) {
return $existingContactsData;
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch syncing missing contacts', [
'teamId' => $this->team->getUuid(),
'total_contacts' => count($contactIds),
'existing_contacts' => count($existingContactsData),
'missing_contacts' => count($missingContactIds),
]);
// Sync missing contacts using batch API
try {
$syncedContactsData = $this->batchSyncCrmObjects('contacts', $missingContactIds);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to sync missing contacts', [
'size' => count($missingContactIds),
'error' => $e->getMessage(),
]);
$syncedContactsData = [];
}
return $existingContactsData + $syncedContactsData;
}
private function batchSyncCrmObjects(string $objectType, array $crmIds): array
{
$syncObjects = [];
$crmObjectIds = array_values($crmIds);
foreach (array_chunk($crmObjectIds, self::BATCH_SIZE) as $chunk) {
try {
$objects = $objectType === 'companies' ?
$this->client->getCompaniesByIds($chunk, $this->getCompanyFields()) :
$this->client->getContactsByIds($chunk, $this->getContactFields());
foreach ($objects as $objectId => $objectData) {
$this->importCrmObject($objectType, (string) $objectId, $objectData, $syncObjects);
}
$this->logger->info('[' . $this->getDisplayName() . '] Batch synced ' . $objectType, [
'requested_count' => count($chunk),
'synced_count' => count($objects),
]);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Batch ' . $objectType . ' sync failed', [
'ids' => $chunk,
'error' => $e->getMessage(),
]);
}
}
return $syncObjects;
}
private function importCrmObject(string $objectType, string $objectId, mixed $objectData, array &$syncObjects): void
{
try {
$object = $objectType === 'companies' ?
$this->importAccount($objectData) :
$this->importContact($objectData);
if ($object) {
$syncObjects[$object->getCrmProviderId()] = $object->getId();
}
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to import batch ' . $objectType, [
'id' => $objectId,
'error' => $e->getMessage(),
]);
}
}
/**
* Prepare associations for a single opportunity
*
* The return value is an array with the following structure:
* [
* 'companies' => [
* $companyCrmId => $companyId,
* ...
* ],
* 'contacts' => [
* $contactCrmId => $contactId,
* ...
* ],
* 'account_id' => $accountId,
* ]
*/
private function prepareAssociationsForOpportunity(
string $oppCrmId,
array $companyAssociations,
array $contactAssociations,
array $associationsData
): array {
$associations = [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
$oppCompanyIds = $companyAssociations[$oppCrmId] ?? [];
foreach ($oppCompanyIds as $companyCrmId) {
if (isset($associationsData['company_id_mappings'][$companyCrmId])) {
$associations['companies'][$companyCrmId] = $associationsData['company_id_mappings'][$companyCrmId];
// Set primary account (first company becomes primary account)
if ($associations['account_id'] === null) {
$associations['account_id'] = $associationsData['company_id_mappings'][$companyCrmId];
}
}
}
$oppContactIds = $contactAssociations[$oppCrmId] ?? [];
foreach ($oppContactIds as $contactCrmId) {
if (isset($associationsData['contact_id_mappings'][$contactCrmId])) {
$associations['contacts'][$contactCrmId] = $associationsData['contact_id_mappings'][$contactCrmId];
}
}
return $associations;
}
/**
* Update only associations for an opportunity
*/
private function updateOpportunityAssociations(Opportunity $opportunity, array $associations): void
{
// Update contact associations
$this->importOpportunityContacts($opportunity, $associations['contacts']);
// Update company (account) associations
$this->updateOpportunityAccount($opportunity, $associations['account_id']);
}
/**
* Remove all contact associations from an opportunity
*/
private function removeAllOpportunityContacts(Opportunity $opportunity): void
{
$currentCount = (int) $opportunity->contacts()->count();
if ($currentCount > 0) {
$opportunity->contacts()->detach();
$this->logger->info('[' . $this->getDisplayName() . '] Removed all contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_count' => $currentCount,
]);
}
}
private function updateOpportunityAccount(Opportunity $opportunity, ?int $accountId): void
{
if ($accountId === null) {
// No account ID provided - keep current account
return;
}
$currentAccountId = $opportunity->getAccountId();
// Only update if account has changed
if ($currentAccountId !== $accountId) {
$opportunity->account_id = $accountId;
$opportunity->save();
$this->logger->info('[' . $this->getDisplayName() . '] Updated opportunity account association', [
'opportunity_id' => $opportunity->getId(),
'old_account_id' => $currentAccountId,
'new_account_id' => $accountId,
]);
}
}
/**
* Find existing opportunities by external IDs (OPTIMIZED VERSION)
* Uses batch query for better performance
*/
private function findExistingOpportunities(array $crmIds): Collection
{
return $this->crmEntityRepository
->findOpportunitiesByExternalIds($this->config, $crmIds);
}
private function processOpportunityBatch(array $opportunities): int
{
$syncedOpportunities = $this->importOpportunityBatch($opportunities);
return count($syncedOpportunities['success'] ?? []);
}
/**
* Convert single deal associations from HubSpot format to internal format
* Handles both HubSpot SDK objects and array formats
*
* @param array $opportunityAssociations Raw associations from HubSpot API or pre-processed
*
* @return array Processed associations with DB IDs
*/
private function convertDealAssociations(array $opportunityAssociations): array
{
$associations = $this->initializeAssociationsStructure();
if (empty($opportunityAssociations)) {
return $associations;
}
$associationIds = $this->extractAssociationIds($opportunityAssociations);
$this->processCompanyAssociations($associationIds, $associations);
$this->processContactAssociations($associationIds, $associations);
return $associations;
}
private function initializeAssociationsStructure(): array
{
return [
'companies' => [],
'contacts' => [],
'account_id' => null, // Primary account for opportunity
];
}
private function extractAssociationIds(array $opportunityAssociations): array
{
$associationIds = [];
foreach ($opportunityAssociations as $type => $associationData) {
if (! empty($associationData)) {
$associationIds[$type] = $this->convertSingleDealAssociations($associationData);
}
}
return $associationIds;
}
private function processCompanyAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['companies'])) {
return;
}
$companyId = $associationIds['companies'][0];
$account = $this->findOrSyncAccount($companyId);
if ($account instanceof Account) {
$associations['companies'][$companyId] = $account->getId();
$associations['account_id'] = $account->getId();
}
}
private function processContactAssociations(array $associationIds, array &$associations): void
{
if (empty($associationIds['contacts'])) {
return;
}
foreach ($associationIds['contacts'] as $contactId) {
$contact = $this->findOrSyncContact($contactId);
if ($contact instanceof Contact) {
$associations['contacts'][$contactId] = $contact->getId();
}
}
}
private function findOrSyncAccount(string $companyId): ?Account
{
$account = $this->crmEntityRepository->findAccountByExternalId($this->config, $companyId);
if (! $account instanceof Account) {
$account = $this->syncAccount($companyId);
}
return $account;
}
private function findOrSyncContact(string $contactId): ?Contact
{
$contact = $this->crmEntityRepository->findContactByExternalId($this->config, $contactId);
if (! $contact instanceof Contact) {
$contact = $this->syncContact($contactId);
}
return $contact;
}
private function convertSingleDealAssociations($opportunityAssociations = null): array
{
$associationData = [];
if ($opportunityAssociations === null) {
return $associationData;
}
// Handle array input (from extractAssociationIds)
if (is_array($opportunityAssociations)) {
return $opportunityAssociations;
}
// Handle CollectionResponseAssociatedId object
if ($opportunityAssociations instanceof CollectionResponseAssociatedId) {
foreach ($opportunityAssociations->getResults() as $association) {
$associationData[] = $association->getId();
}
}
return $associationData;
}
private function importOrUpdateOpportunity($crmData, ?bool $exists = null): ?Opportunity
{
if (empty($crmData['properties'])) {
return null;
}
$properties = $crmData['properties'];
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = (string) $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile($ownerId);
}
$associations = $crmData['associations'] ?? [];
$accountId = $this->resolveAccountId($associations);
// Only fetch account if we need it for user_id fallback
$accountUserId = null;
if ($profile?->getUserId() === null && $accountId !== null) {
$accountUserId = $this->crmEntityRepository
->findAccountByConfigurationAndId(
$this->config,
$accountId
)?->getUserId();
}
$crmId = (string) $crmData['id'];
if (($profile?->getUserId() === null) && ($accountUserId === null)) {
$this->logger->error(
'[HubSpot] Skip import, no user_id found',
[
'id' => $crmId,
]
);
return null;
}
$opportunityExists = $exists ?? (bool) $this->crmEntityRepository->findOpportunityByExternalId(
$this->config,
$crmId
);
if ($opportunityExists) {
return $this->updateOpportunity(
$crmId,
$properties,
$associations,
$accountUserId
);
}
return $this->createOpportunity(
$crmId,
$properties,
$associations,
$accountUserId,
);
}
/**
* Create new opportunity
*/
private function createOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): ?Opportunity {
$accountId = $this->resolveAccountId($associations);
if (! $accountId) {
return null;
}
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
if (! $businessProcess) {
return null;
}
$stage = $this->resolveStage($businessProcess, $properties['dealstage'] ?? null);
if (! $stage) {
return null;
}
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->importOpportunityContacts($opportunity, $associations['contacts']);
if ($opportunity->wasRecentlyCreated) {
MatchActivitiesToNewOpportunity::dispatch($opportunity->getId());
}
return $opportunity;
}
/**
* Update existing opportunity
*/
private function updateOpportunity(
string $crmId,
array $properties,
array $associations,
?int $accountUserId = null
): Opportunity {
$accountId = $this->resolveAccountId($associations);
$businessProcess = $this->resolveBusinessProcess($properties['pipeline'] ?? null);
$stage = $businessProcess ? $this->resolveStage($businessProcess, $properties['dealstage'] ?? null) : null;
$data = $this->buildOpportunityData(
$properties,
$accountId,
$businessProcess,
$stage,
$accountUserId
);
$attributes = [
'crm_configuration_id' => $this->config->getId(),
'crm_provider_id' => $crmId,
];
$values = array_merge($attributes, $data);
$opportunity = $this->crmEntityRepository->upsertOpportunity($attributes, $values);
$this->importExternalFieldData($properties, $opportunity->getId());
$this->updateOpportunityAssociations($opportunity, $associations);
return $opportunity;
}
private function resolveAccountId(array $associations): ?int
{
if (! empty($associations['account_id'])) {
return $associations['account_id'];
}
if (empty($associations)) {
return null;
}
// Fallback: use first company as account (currently SDK returns one company)
foreach ($associations['companies'] as $accountId) {
return $accountId;
}
return null;
}
private function buildOpportunityData(
array $properties,
?int $accountId,
?BusinessProcess $businessProcess,
?Stage $stage,
?int $accountUserId = null
): array {
$ownerId = null;
$profile = null;
if (! empty($properties['hubspot_owner_id'])) {
$ownerId = $properties['hubspot_owner_id'];
$profile = $this->getCachedOwnerProfile((string) $ownerId);
}
$userId = $profile?->getUserId() ?? $accountUserId;
$name = 'Unknown';
if (isset($properties['dealname'])) {
$name = mb_strimwidth($properties['dealname'], 0, 128);
}
$amount = $this->resolveAmount($properties);
$currency = $properties['deal_currency_code'] ?? null;
$closeDate = null;
if (! empty($properties['closedate'])) {
$closeDate = Carbon::parse($properties['closedate'])->format('Y-m-d');
}
$remotelyCreatedAt = null;
if (! empty($properties['createdate']) && strtotime($properties['createdate'])) {
$date = $this->parseCleanDatetime($properties['createdate']);
$remotelyCreatedAt = $date?->format('Y-m-d H:i:s');
}
$closedStages = $this->getClosedDealStages();
$isWon = in_array($properties['dealstage'], $closedStages['won']);
$isLost = in_array($properties['dealstage'], $closedStages['lost']);
$data = [
'team_id' => $this->team->getId(),
'user_id' => $userId,
'owner_id' => $ownerId,
'name' => $name,
'value' => ! empty($amount) ? $amount : null,
'currency_code' => CurrencyFormatter::formatCode($currency),
'close_date' => $closeDate,
'is_closed' => $isWon || $isLost,
'is_won' => $isWon,
'remotely_created_at' => $remotelyCreatedAt,
'probability' => $this->resolveDealProbability($properties['hs_deal_stage_probability']),
'forecast_category' => $this->resolveForecastCategory($properties['hs_manual_forecast_category']),
];
if ($accountId) {
$data['account_id'] = $accountId;
}
if ($stage) {
$data['stage_id'] = $stage->id;
}
if ($businessProcess) {
$recordType = $this->getCachedBusinessProcessRecordType($businessProcess);
if ($recordType) {
$data['record_type_id'] = $recordType->id;
}
}
return $data;
}
private function getCachedOwnerProfile(string $ownerId): ?Profile
{
$cacheKey = $this->config->getId() . ':' . $ownerId;
if (array_key_exists($cacheKey, $this->cachedOwnerProfiles)) {
return $this->cachedOwnerProfiles[$cacheKey];
}
$profile = $this->crmEntityRepository->findProfileByExternalId($this->config, $ownerId);
$this->cachedOwnerProfiles[$cacheKey] = $profile;
return $profile;
}
private function getCachedBusinessProcessRecordType(BusinessProcess $businessProcess): mixed
{
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId();
if (array_key_exists($cacheKey, $this->cachedRecordTypes)) {
return $this->cachedRecordTypes[$cacheKey];
}
$recordType = $this->crmEntityRepository->getBusinessProcessRecordType($businessProcess);
$this->cachedRecordTypes[$cacheKey] = $recordType;
return $recordType;
}
private function resolveBusinessProcess(?string $pipelineId): ?BusinessProcess
{
if ($pipelineId === null) {
return null;
}
$cacheKey = $this->getBusinessProcessCacheKey($pipelineId);
if (isset($this->cachedBusinessProcesses[$cacheKey])) {
return $this->cachedBusinessProcesses[$cacheKey];
}
$businessProcess = $this->getBusinessProcess($pipelineId);
if (! $businessProcess instanceof BusinessProcess) {
$this->importStages();
$this->cachedClosedDealStages = null;
$businessProcess = $this->getBusinessProcess($pipelineId);
}
if (! $businessProcess instanceof BusinessProcess) {
$this->logger->info(
'[HubSpot] Deal is not attached to a pipeline',
[
'pipeline' => $pipelineId]
);
}
$this->cachedBusinessProcesses[$cacheKey] = $businessProcess;
return $businessProcess;
}
private function getBusinessProcess(string $pipelineId): ?BusinessProcess
{
return $this->crmEntityRepository->findBusinessProcessesByExternalId($this->config, $pipelineId);
}
private function getBusinessProcessCacheKey(string $pipelineId): string
{
return $this->config->getId() . '_' . $pipelineId;
}
private function resolveStage(BusinessProcess $businessProcess, ?string $stageId): ?Stage
{
if (empty($stageId)) {
return null;
}
$cacheKey = $this->config->getId() . ':' . $businessProcess->getId() . ':' . $stageId;
if (isset($this->cachedStages[$cacheKey])) {
return $this->cachedStages[$cacheKey];
}
$stage = $this->crmEntityRepository->getPipelineStageByConditions(
$businessProcess,
[
'crm_provider_id' => $stageId,
'type' => Stage::TYPE_OPPORTUNITY,
]
);
if ($stage === null) {
$this->importStages(null, $stageId);
$this->cachedClosedDealStages = null;
}
if ($stage === null) {
$this->logger->info('[HubSpot] Stage does not exist => ' . $stageId);
}
$this->cachedStages[$cacheKey] = $stage;
return $stage;
}
private function resolveAmount(array $properties): ?string
{
$amount = null;
if (! empty($properties['amount'])) {
$amount = str_replace(',', '', $properties['amount']);
}
if ($this->config->hasDefaultCurrencyFieldSet()) {
$valueFieldName = $this->config->getDefaultCurrencyField()->getCrmProviderId();
$amount = $properties[$valueFieldName] ?? $amount;
}
return $amount;
}
private function parseCleanDatetime(string $datetime): ?Carbon
{
// Treat pre-1980 values as invalid
$minValidDate = Carbon::parse('1980-01-01 00:00:00');
try {
$date = Carbon::parse($datetime);
if ($minValidDate->gt($date)) {
return null;
}
return $date;
} catch (Exception) {
return null; // On parse error, treat as null
}
}
private function resolveDealProbability(?string $stageProbability): int
{
if ($stageProbability === null) {
return 0;
}
$probability = (float) $stageProbability;
return $probability > 1 ? 0 : (int) ($probability * 100);
}
private function resolveForecastCategory(?string $forecastCategory): string
{
if (! $forecastCategory) {
return Forecast::FORECAST_CATEGORY_UNCATEGORIZED;
}
$forecastCategory = str_replace('_', ' ', $forecastCategory);
return ucwords(strtolower($forecastCategory));
}
private function importExternalFieldData(array $properties, int $opportunityId): void
{
$this->importOpportunityCrmFieldData(
$properties,
$this->getCachedOpportunitySyncableFields(),
$opportunityId
);
}
private function getCachedOpportunitySyncableFields(): array
{
$cacheKey = (string) $this->config->getId();
if (! isset($this->cachedOpportunitySyncableFields[$cacheKey])) {
$this->cachedOpportunitySyncableFields[$cacheKey] = $this->getOpportunitySyncableFields();
}
return $this->cachedOpportunitySyncableFields[$cacheKey];
}
private function importOpportunityContacts(Opportunity $opportunity, array $associations): void
{
// Handle empty or missing contact associations
if (empty($associations)) {
// Remove all existing contact associations if none provided
$this->removeAllOpportunityContacts($opportunity);
return;
}
// Use differential sync approach for better performance and accuracy
$this->syncOpportunityContactsDifferential($opportunity, $associations);
}
/**
* Sync opportunity contacts using differential approach
* This compares current vs new associations and only makes necessary changes
*/
private function syncOpportunityContactsDifferential(Opportunity $opportunity, array $contactAssociations): void
{
$currentContactCrmIds = $this->getCurrentContactCrmIds($opportunity);
$contactAssociationIds = array_keys($contactAssociations);
$contactsToAdd = array_diff($contactAssociationIds, $currentContactCrmIds);
$contactsToRemove = array_diff($currentContactCrmIds, $contactAssociationIds);
if (empty($contactsToAdd) && empty($contactsToRemove)) {
return;
}
$this->logContactAssociationChanges($opportunity, $currentContactCrmIds, $contactAssociations, $contactsToAdd, $contactsToRemove);
$this->removeContactAssociations($opportunity, $contactsToRemove);
$this->addContactAssociations($opportunity, $contactsToAdd, $contactAssociations);
}
private function getCurrentContactCrmIds(Opportunity $opportunity): array
{
return $opportunity->contacts()
->pluck('contacts.crm_provider_id')
->toArray();
}
private function logContactAssociationChanges(
Opportunity $opportunity,
array $currentContactCrmIds,
array $contactAssociations,
array $contactsToAdd,
array $contactsToRemove
): void {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association changes', [
'opportunity_id' => $opportunity->getId(),
'current_contacts' => $currentContactCrmIds,
'new_contacts' => $contactAssociations,
'contacts_to_add' => $contactsToAdd,
'contacts_to_remove' => $contactsToRemove,
]);
}
private function removeContactAssociations(Opportunity $opportunity, array $contactsToRemove): void
{
if (empty($contactsToRemove)) {
return;
}
$contactsToDetach = $opportunity->contacts()
->whereIn('contacts.crm_provider_id', $contactsToRemove)
->pluck('contacts.id')
->toArray();
if (! empty($contactsToDetach)) {
$opportunity->contacts()->detach($contactsToDetach);
$this->logger->info('[' . $this->getDisplayName() . '] Removed contact associations', [
'opportunity_id' => $opportunity->getId(),
'removed_contact_crm_ids' => $contactsToRemove,
'removed_contact_count' => count($contactsToDetach),
]);
}
}
private function addContactAssociations(Opportunity $opportunity, array $contactsToAdd, array $contactAssociations): void
{
if (empty($contactsToAdd)) {
return;
}
$contactsAdded = [];
foreach ($contactsToAdd as $crmId) {
$id = $contactAssociations[$crmId];
if ($this->attachSingleContact($opportunity, (string) $crmId, $id)) {
$contactsAdded[] = $crmId;
}
}
$this->logAddedContacts($opportunity, $contactsAdded);
}
private function attachSingleContact(Opportunity $opportunity, string $crmId, int $id): bool
{
try {
return $this->performContactAttachment($opportunity, $id, $crmId);
} catch (\Throwable $e) {
$this->logger->warning('[' . $this->getDisplayName() . '] Failed to add contact association', [
'opportunity_id' => $opportunity->getId(),
'contact_crm_id' => $crmId,
'error' => $e->getMessage(),
]);
return false;
}
}
private function performContactAttachment(Opportunity $opportunity, int $contactId, string $crmId): bool
{
try {
$opportunity->contacts()->attach($contactId, [
'crm_provider_id' => $crmId,
]);
return true;
} catch (\Illuminate\Database\QueryException $e) {
if (str_contains($e->getMessage(), 'Duplicate entry')) {
$this->logger->info('[' . $this->getDisplayName() . '] Contact association already exists', [
'contact_id' => $contactId,
'contact_crm_id' => $crmId,
'opportunity_id' => $opportunity->getId(),
]);
return false;
}
throw $e;
}
}
private function logAddedContacts(Opportunity $opportunity, array $contactsAdded): void
{
if (! empty($contactsAdded)) {
$this->logger->info('[' . $this->getDisplayName() . '] Added contact associations', [
'opportunity_id' => $opportunity->getId(),
'added_contact_crm_ids' => $contactsAdded,
'added_contacts_count' => count($contactsAdded),
]);
}
}
}
Execute
Explain Plan
Browse Query History
View Parameters
Open Query Execution Settings…
In-Editor Results
Tx: Auto
Cancel Running Statements
Playground
jiminny
Code changed:
Hide
Sync Changes
Hide This Notification
31
9
29
3
112
Previous Highlighted Error
Next Highlighted Error
SELECT * FROM team_features where team_id = 1;
SELECT * FROM teams WHERE name LIKE '%Vixio%'; # 340,270,11922
SELECT * FROM users WHERE team_id = 340; # 12015
select * from social_accounts sa
join users u on sa.sociable_id = u.id
where u.team_id = 340
and sa.provider = 'salesforce';
# and sa.provider = 'salesloft';
select * from crm_fields where crm_configuration_id = 270 and object_type = 'event';
# 125558 - Event Type - Event_Type__c
# 125552 - Event Status - Event_Status__c
SELECT * FROM sidekick_settings WHERE team_id = 340;
SELECT * FROM crm_field_values WHERE crm_field_id in (125552);
select * from activities where crm_configuration_id = 270
and type = 'conference' and crm_provider_id IS NOT NULL
and actual_start_time > '2024-09-16 09:00:00' order by scheduled_start_time;
SELECT * FROM activities WHERE id = 20871677;
SELECT * FROM crm_field_data WHERE activity_id = 20871677;
select * from crm_layouts where crm_configuration_id = 270;
select * from crm_layout_entities where crm_layout_id in (886,887);
SELECT * FROM crm_configurations WHERE id = 270;
...
|
PhpStorm
|
faVsco.js – console [EU]
|
NULL
|
89068
|