I have a working plugin
That imports data from CSV and inserts it in the structure field
I also have a files field inside the structure and wish to upload images from external sources, create files inside the page, and attach them to the structure field. The main problem is that if more than 100 items in the file I have an error with response time … Right now it is working with manual batches (that I select in other field) and I need to re save page until all images are uploaded.
My question is, is there any solution to do an image download, and attach to the structure field as a background process … where should I look for modification?
P.S. It will be a great idea to create a plugin that will help export/import any data to KirbyCMS as it is in Wordpress or other CMS solutions.
<?php
Kirby::plugin('ggg/gaming-software', [
'hooks' => [
'site.update:after' => function () {
createGamingSoftwarePage();
},
'page.update:after' => function ($newPage, $oldPage) {
processAndDownload($newPage);
}
],
'blueprints' => [
'pages/gaming-software' => __DIR__ . '/blueprints/pages/gaming-software.yml',
],
'templates' => [
'gaming-software' => __DIR__ . '/templates/gaming-software.php',
]
]);
function createGamingSoftwarePage() {
$site = kirby()->site();
if (!$site->find('gaming-software')) {
try {
$gamingSoftwarePage = $site->createChild([
'slug' => 'gaming-software',
'template' => 'gaming-software',
'content' => [
'title' => 'Gaming Software'
]
]);
} catch (Exception $e) {
throw new Exception('Error creating gaming software page: ' . $e->getMessage());
}
if (isset($gamingSoftwarePage)) {
try {
$gamingSoftwarePage->changeStatus('unlisted');
} catch (Exception $e) {
throw new Exception('Error listing gaming software page: ' . $e->getMessage());
}
}
}
}
function processAndDownload($page) {
$fileFieldKey = 'gaming_software_files';
$structureFieldKey = 'gaming_software';
$mapping = [
'Software' => 'software_name',
'Logo' => 'software_logo_temp'
];
$fileField = $page->content()->get($fileFieldKey);
$jsonArray = $page->content()->get($structureFieldKey)->yaml() ?? [];
$existingSoftwareNames = array_column($jsonArray, 'software_name');
// Process CSV files
if ($fileField->isNotEmpty()) {
$fileIds = $fileField->toFiles();
foreach ($fileIds as $file) {
if ($file->extension() === 'csv') {
$csvData = array_map('str_getcsv', file($file->root()));
$headers = array_shift($csvData);
foreach ($csvData as $row) {
$item = array_combine($headers, $row);
$mappedItem = [];
foreach ($mapping as $csvKey => $structureKey) {
$mappedItem[$structureKey] = $item[$csvKey] ?? '';
}
// Check for duplicates before adding
if (!in_array($mappedItem['software_name'], $existingSoftwareNames)) {
$mappedItem['_key'] = $item['id'] ?? uniqid();
$jsonArray[] = $mappedItem;
$existingSoftwareNames[] = $mappedItem['software_name'];
}
}
}
}
}
// Download images
$batchSize = (int)$page->content()->get('import_items')->value(); // Get the number of items to process per iteration
$count = 0;
foreach ($jsonArray as &$item) {
if (!empty($item['software_logo_temp']) && empty($item['software_logo'])) {
$urlField = $item['software_logo_temp'];
$softwareName = $item['software_name'] ?? 'unknown';
$fileId = downloadAndAttachFile($urlField, $softwareName, $page);
if ($fileId) {
$item['software_logo'] = $fileId;
unset($item['software_logo_temp']);
$count++;
}
}
if ($count >= $batchSize) {
break;
}
}
// Encode data to YAML format
$yamlData = yaml::encode($jsonArray);
// Update the page with the combined data
try {
$page->update([
$structureFieldKey => $yamlData,
'gaming_software_files' => '' // Clear the file field if necessary
]);
} catch (Exception $e) {
throw new Exception('Error updating fields: ' . $e->getMessage());
}
}
function downloadAndAttachFile($urlField, $softwareName, $page) {
if (preg_match('/\((https?:\/\/[^\)]+)\)$/', $urlField, $matches)) {
$url = $matches[1];
$parsedUrl = parse_url($url);
$pathParts = pathinfo($parsedUrl['path']);
$extension = strtolower($pathParts['extension']);
$validExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg'];
if (!in_array($extension, $validExtensions)) {
return ''; // Invalid file type, ignore
}
$filename = str_replace(' ', '_', strtolower($softwareName)) . '_logo.' . $extension;
// Temporary file path within the page directory
$tempFilePath = $page->root() . '/temp-' . $filename;
// Download file to temporary path
try {
file_put_contents($tempFilePath, file_get_contents($url));
} catch (Exception $e) {
// Log the error and return an empty string
error_log('Failed to download file: ' . $e->getMessage());
return '';
}
// Move the file to the content directory
try {
$file = $page->createFile([
'source' => $tempFilePath,
'template' => 'image', // You can specify a different template if needed
'filename' => $filename,
]);
// Clean up the temporary file
unlink($tempFilePath);
return $file->id();
} catch (Exception $e) {
// Log the error and return an empty string
error_log('Failed to create file: ' . $e->getMessage());
return '';
}
}
return '';
}```