I am implementing a way to import content into Drupal 8 via a CSV. This CSV contains various fields such as node ID, title, body, etc. It has been working great but I am very concerned about how long this takes to run. When there are quite a few rows, it starts taking minutes to complete.
This was a community module I found that was no longer working, and I have spent some time bringing it up to speed with Drupal 8.
ContentImport.php
<?php
namespace Drupal\contentimport\Form;
use Drupal\contentimport\Controller\ContentImportController;
use Drupal\Core\Ajax\AjaxResponse;
use Drupal\Core\Ajax\HtmlCommand;
use Drupal\Core\Form\ConfigFormBase;
use Drupal\Core\Form\FormStateInterface;
use Drupal\file\Entity\File;
use Drupal\node\Entity\Node;
use Drupal\taxonomy\Entity\Term;
use Drupal\taxonomy\Entity\Vocabulary;
use Drupal\user\Entity\User;
/**
* Configure Content Import settings for this site.
*/
class ContentImport extends ConfigFormBase
{
/**
* {@inheritdoc}
*/
public function getFormId()
{
return 'contentimport';
}
/**
* @param $contentType
* @param $fieldNames
* @param $fieldTypes
* @param $images
* @param $data
* @param $keyIndex
* @param $nodeArray
* @param $fieldSettings
* @return array
*/
public function buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings)
{
$node_id = [];
for ($f = 0; $f < count($fieldNames); $f++) {
switch ($fieldTypes[$f]) {
case 'image':
break;
case 'file':
if (!empty($images[$data[$keyIndex[$fieldNames[$f]]]])) {
$nodeArray[$fieldNames[$f]] = [['target_id' => $images[$data[$keyIndex[$fieldNames[$f]]]]->id()]];
}
break;
case 'entity_reference':
if ($fieldSettings[$f]['target_type'] == 'taxonomy_term') {
$reference = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
if (is_array($reference) && $reference[0] != '') {
$terms = ContentImport::getTermReference($reference[0], $reference[1]);
$nodeArray[$fieldNames[$f]] = $terms;
}
} elseif ($fieldSettings[$f]['target_type'] == 'user') {
$userArray = explode(', ', $data[$keyIndex[$fieldNames[$f]]]);
$users = ContentImport::getUserInfo($userArray);
$nodeArray[$fieldNames[$f]] = $users;
} elseif ($fieldSettings[$f]['target_type'] == 'node') {
if (isset($data[$keyIndex[$fieldNames[$f]]]) && !empty($data[$keyIndex[$fieldNames[$f]]])) {
$nids = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
$fieldName = $fieldNames[$f];
$nids = $this->check_node_id($nids, $contentType, $fieldName);
$nodeArray[$fieldNames[$f]] = $nids;
}
} else {
$nodeArray[$fieldNames[$f]] = '';
}
break;
case 'entity_reference_revisions':
// TODO Implement this field type
break;
case 'text_with_summary':
// TODO Implement this field type
break;
case 'text_long':
$nodeArray[$fieldNames[$f]] = ['value' => $data[$keyIndex[$fieldNames[$f]]], 'format' => 'full_html'];
break;
case 'url':
if (!empty($data[$keyIndex[$fieldNames[$f]]])) {
$nodeArray['path']['alias'] = $data[$keyIndex[$fieldNames[$f]]];
} else {
$nodeArray['path']['alias'] = "/node/" . $node_id["value"];
}
break;
case 'text':
$nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
break;
case 'nid':
$node_id = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
break;
case 'datetime':
if ($data[$keyIndex[$fieldNames[$f]]] != '') {
$dateTimeStamp = strtotime($data[$keyIndex[$fieldNames[$f]]]);
$newDateString = date('Y-m-d', $dateTimeStamp);
}
$nodeArray[$fieldNames[$f]] = ["value" => $newDateString];
break;
case 'timestamp':
$nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
break;
case 'boolean':
$nodeArray[$fieldNames[$f]] = ($data[$keyIndex[$fieldNames[$f]]] == 'On' || $data[$keyIndex[$fieldNames[$f]]] == 'Yes') ? 1 : 0;
break;
case 'geolocation':
$geoArray = explode(";", $data[$keyIndex[$fieldNames[$f]]]);
if (count($geoArray) > 0) {
$geoMultiArray = [];
for ($g = 0; $g < count($geoArray); $g++) {
$latlng = explode(",", $geoArray[$g]);
for ($l = 0; $l < count($latlng); $l++) {
$latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
}
array_push($geoMultiArray, [
'lat' => $latlng[0],
'lng' => $latlng[1],
]);
}
$nodeArray[$fieldNames[$f]] = $geoMultiArray;
} else {
$latlng = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
for ($l = 0; $l < count($latlng); $l++) {
$latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
}
$nodeArray[$fieldNames[$f]] = ['lat' => $latlng[0], 'lng' => $latlng[1]];
}
break;
case 'entity_reference_revisions':
// TODO Implement this field type
break;
default:
$nodeArray[$fieldNames[$f]] = $data[$keyIndex[$fieldNames[$f]]];
break;
}
}
$nodeArray['type'] = strtolower($contentType);
$nodeArray['uid'] = 1;
$nodeArray['promote'] = 0;
$nodeArray['langcode'] = 'en';
$nodeArray['sticky'] = 0;
return array($node_id, $nodeArray, $data);
}
/**
* {@inheritdoc}
*/
protected function getEditableConfigNames()
{
return [
'contentimport.settings',
];
}
/**
* Content Import Form.
*/
public function buildForm(array $form, FormStateInterface $form_state)
{
$contentTypes = ContentImportController::getAllContentTypes();
$form['contentimport_contenttype'] = [
'#type' => 'select',
'#title' => $this->t('Select Content Type'),
'#options' => $contentTypes,
'#default_value' => t('Select'),
'#required' => true,
'#ajax' => [
'event' => 'change',
'callback' => '::contentImportcallback',
'wrapper' => 'content_import_fields_change_wrapper',
'progress' => [
'type' => 'throbber',
'message' => null,
],
],
];
$form['file_upload'] = [
'#type' => 'file',
'#title' => $this->t('Import CSV File'),
'#size' => 40,
'#description' => $this->t('Select the CSV file to be imported.'),
'#required' => false,
'#autoupload' => true,
'#upload_validators' => ['file_validate_extensions' => ['csv']],
];
$form['import_ct_markup'] = [
'#suffix' => '<div id="content_import_fields_change_wrapper"></div>
<div id="content_submit_change_wrapper"></div>',
];
$form['submit'] = [
'#type' => 'submit',
'#value' => $this->t('Import'),
'#button_type' => 'primary',
'#ajax' => [
'event' => 'click',
'callback' => '::submitForm',
'wrapper' => 'content_submit_change_wrapper',
'progress' => [
'type' => 'throbber',
'message' => 'IMPORTING PLEASE WAIT',
],
],
];
$form['import_ct_markup2'] = [
'#suffix' => '<div id="content_submit_change_success_wrapper"></div>',
];
$form['import_ct_markup3'] = [
'#suffix' => '<div id="content_submit_change_fail_wrapper"></div>',
];
return parent::buildForm($form, $form_state);
}
/**
* @param $nids Titles of the node referenced
* @param $contentType Dropdown from the form to select content type
* @param $fieldName
* @return array Array of node IDs
*/
public function check_node_id($nids, $contentType, $fieldName)
{
$referencedEntity = '';
foreach (\Drupal::entityManager()->getFieldDefinitions('node', $contentType) as $field_definition) {
if (!empty($field_definition->getTargetBundle() && $field_definition->getName() == $fieldName)) {
$referencedEntity .= reset($field_definition->getSettings()['handler_settings']['target_bundles']);
}
}
$loaded_nids = [];
if (is_array($nids)) {
foreach ($nids as $nid) {
$nid = trim($nid);
if (Node::load($nid)) {
$loaded_nids[] = $nid;
} else {
drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nid\""), 'error');
}
}
} else {
$nids = trim($nids);
if (Node::load($nids)) {
$loaded_nids[] = $nids;
} else {
drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nids\""), 'error');
}
}
return $loaded_nids;
}
/**
* Content Import Sample CSV Creation.
*/
public function contentImportcallback(array &$form, FormStateInterface $form_state)
{
global $base_url;
$ajax_response = new AjaxResponse();
$contentType = $form_state->getValue('contentimport_contenttype');
$fields = ContentImport::getFields($contentType);
$fieldArray = $fields['name'];
$contentTypeFields = 'node_id,';
$contentTypeFields .= 'title,';
$contentTypeFields .= 'path,';
foreach ($fieldArray as $key => $val) {
$contentTypeFields .= $val . ',';
}
$contentTypeFields = substr($contentTypeFields, 0, -1);
$sampleFile = $contentType . '.csv';
$handle = fopen("sites/default/files/" . $sampleFile, "w+") or die("There is no permission to create log file. Please give permission for sites/default/file!");
fwrite($handle, $contentTypeFields);
$ajax_response->addCommand(new HtmlCommand('#content_import_fields_change_wrapper', null));
return $ajax_response;
}
/**
* Content Import Form Submission.
*/
public function submitForm(array &$form, FormStateInterface $form_state)
{
$ajax_response = new AjaxResponse();
$contentType = $form_state->getValue('contentimport_contenttype');
ContentImport::updateNode($_FILES, $contentType);
if ($ajax_response->isSuccessful()) {
$ajax_response->addCommand(new HtmlCommand('#content_submit_change_success_wrapper', 'The import was successful.'));
return $ajax_response;
} else {
$ajax_response->addCommand(new HtmlCommand('#content_submit_change_fail_wrapper', 'The import was NOT successful.'));
return $ajax_response;
}
}
/**
* To get all Content Type Fields.
*/
public function getFields($contentType)
{
$fields = [];
foreach (\Drupal::entityManager()
->getFieldDefinitions('node', $contentType) as $field_definition) {
if (!empty($field_definition->getTargetBundle())) {
$fields['name'][] = $field_definition->getName();
$fields['type'][] = $field_definition->getType();
$fields['setting'][] = $field_definition->getSettings();
}
}
return $fields;
}
/**
* To get Reference field ids.
*/
public function getTermReference($voc, $terms)
{
$vocName = strtolower($voc);
$vid = preg_replace('@[^a-z0-9_]+@', '_', $vocName);
$vocabularies = Vocabulary::loadMultiple();
/* Create Vocabulary if does not exist */
if (!isset($vocabularies[$vid])) {
ContentImport::createVoc($vid, $voc);
}
$termArray = array_map('trim', explode(',', $terms));
$termIds = [];
foreach ($termArray as $term) {
$term_id = ContentImport::getTermId($term, $vid);
if (empty($term_id)) {
$term_id = ContentImport::createTerm($voc, $term, $vid);
}
$termIds[]['target_id'] = $term_id;
}
return $termIds;
}
/**
* Create Vocabulary if not available.
*/
public function createVoc($vid, $voc)
{
$vocabulary = Vocabulary::create([
'vid' => $vid,
'machine_name' => $vid,
'name' => $voc,
]);
$vocabulary->save();
}
/**
* Create Term if it is not available.
*/
public function createTerm($voc, $term, $vid)
{
Term::create([
'parent' => [$voc],
'name' => $term,
'vid' => $vid,
])->save();
$termId = ContentImport::getTermId($term, $vid);
return $termId;
}
/**
* Get Term id
*/
public function getTermId($term, $vid)
{
$termRes = db_query('SELECT n.tid FROM {taxonomy_term_field_data} n WHERE n.name = :uid AND n.vid = :vid', [':uid' => $term, ':vid' => $vid]);
foreach ($termRes as $val) {
$term_id = $val->tid;
}
return $term_id;
}
/**
* Get user information based on emailIds.
*/
public static function getUserInfo($userArray)
{
$uids = [];
foreach ($userArray as $usermail) {
if (filter_var($usermail, FILTER_VALIDATE_EMAIL)) {
$users = \Drupal::entityTypeManager()->getStorage('user')
->loadByProperties([
'mail' => $usermail,
]);
} else {
$users = \Drupal::entityTypeManager()->getStorage('user')
->loadByProperties([
'name' => $usermail,
]);
}
$user = reset($users);
if ($user) {
$uids[] = $user->id();
} else {
$user = User::create();
$user->uid = '';
$user->setUsername($usermail);
$user->setEmail($usermail);
$user->set("init", $usermail);
$user->enforceIsNew();
$user->activate();
$user->save();
$users = \Drupal::entityTypeManager()->getStorage('user')
->loadByProperties(['mail' => $usermail]);
$uids[] = $user->id();
}
}
return $uids;
}
/**
* To import data as Content type nodes.
*/
public function updateNode($filedata, $contentType)
{
global $base_url;
$fields = ContentImport::getFields($contentType);
$fieldNames = $fields['name'];
$fieldTypes = $fields['type'];
$fieldSettings = $fields['setting'];
$files = glob('sites/default/files/' . $contentType . '/images/*.*');
$images = [];
foreach ($files as $file_name) {
$image = File::create(['uri' => 'public://' . $contentType . '/images/' . basename($file_name)]);
$image->save();
$images[basename($file_name)] = $image;
}
$mimetype = 1;
if ($mimetype) {
$location = $filedata['files']['tmp_name']['file_upload'];
if (($handle = fopen($location, "r")) !== false) {
$keyIndex = [];
$index = 0;
while (($data = fgetcsv($handle)) !== false) {
$index++;
if ($index < 2) {
array_push($fieldNames, 'node_id');
array_push($fieldTypes, 'nid');
array_push($fieldNames, 'title');
array_push($fieldTypes, 'text');
array_push($fieldNames, 'path');
array_push($fieldTypes, 'url');
foreach ($fieldNames as $fieldValues) {
$i = 0;
foreach ($data as $dataValues) {
if ($fieldValues == $dataValues) {
$keyIndex[$fieldValues] = $i;
}
$i++;
}
}
continue;
}
$nodeArray = [];
list($node_id, $nodeArray, $data) = $this->buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings);
if ($nodeArray['title']['value'] != '') {
if (empty($node_id["value"])) {
$node = Node::create($nodeArray);
$node->save();
} elseif (is_null(Node::load($node_id["value"]))) {
drupal_set_message($this->t("NO NODE USING ID " . $node_id['value']), 'error');
} else {
foreach ($nodeArray as $key => $value) {
$node = Node::load($node_id["value"]);
$node->set($key, $value);
$node->save();
}
}
}
}
fclose($handle);
}
}
}
}
ContentImportController.php
<?php
namespace Drupal\contentimport\Controller;
use Drupal\Core\Controller\ControllerBase;
/**
* Controller routines for contentimport routes.
*/
class ContentImportController extends ControllerBase {
/**
* Get All Content types.
*/
public static function getAllContentTypes() {
$contentTypes = \Drupal::service('entity.manager')->getStorage('node_type')->loadMultiple();
$contentTypesList = [];
$contentTypesList['none'] = 'Select';
foreach ($contentTypes as $contentType) {
$contentTypesList[$contentType->id()] = $contentType->label();
}
return $contentTypesList;
}
}
