From 4a05fcf026f1fd8d87328d57bea780617a85903b Mon Sep 17 00:00:00 2001 From: jp31415926 Date: Sat, 5 Jan 2013 01:58:04 -0600 Subject: [PATCH 1/2] Added ImportIndex annotation and support for updating DB instead of just adding records --- Annotation/ImportIndex.php | 10 ++++ Controller/ImportController.php | 13 +++-- Import/Importer.php | 58 ++++++++++++++++++----- Resources/views/Import/mapping.html.twig | 5 +- Util/.AbstractBase.php.swp | Bin 12288 -> 0 bytes Util/.Writer.php.swp | Bin 12288 -> 0 bytes Util/FieldRetriever.php | 10 +++- Util/Reader.php | 12 +++-- 8 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 Annotation/ImportIndex.php delete mode 100644 Util/.AbstractBase.php.swp delete mode 100644 Util/.Writer.php.swp diff --git a/Annotation/ImportIndex.php b/Annotation/ImportIndex.php new file mode 100644 index 0000000..d173f7e --- /dev/null +++ b/Annotation/ImportIndex.php @@ -0,0 +1,10 @@ +container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); + list($fieldChoices, $fieldIndexes) = $this->container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); $form = $this->container->get('form.factory')->create(new ImportFormType(), null, array('field_choices' => $fieldChoices)); @@ -49,8 +49,10 @@ public function uploadAction($alias) */ public function mappingAction(Request $request, $alias) { - $fieldChoices = $this->container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); + list($fieldChoices, $fieldIndexes) = $this->container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); + // prepend blank choice into choices array + array_unshift($fieldChoices, ''); $form = $this->container->get('form.factory')->create(new ImportFormType(), null, array('field_choices' => $fieldChoices)); if ('POST' == $request->getMethod()) { @@ -94,8 +96,9 @@ public function mappingAction(Request $request, $alias) */ public function processAction(Request $request, $alias) { - $fieldChoices = $this->container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); - + list($fieldChoices, $fieldIndexes) = $this->container->get('avro_csv.field_retriever')->getFields($this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias)), 'title', true); + // prepend blank choice into choices array + array_unshift($fieldChoices, ''); $form = $this->container->get('form.factory')->create(new ImportFormType(), null, array('field_choices' => $fieldChoices)); if ('POST' == $request->getMethod()) { @@ -106,7 +109,7 @@ public function processAction(Request $request, $alias) $importer->init(sprintf('%s%s', $this->container->getParameter('avro_csv.tmp_upload_dir'), $form['filename']->getData()), $this->container->getParameter(sprintf('avro_csv.objects.%s.class', $alias), $form['delimiter']->getData())); - $importer->import($form['fields']->getData()); + $importer->import($form['fields']->getData(), $fieldIndexes); $this->container->get('session')->getFlashBag()->set('success', $importer->getImportCount().' items imported.'); diff --git a/Import/Importer.php b/Import/Importer.php index 138fb0a..cdf3548 100644 --- a/Import/Importer.php +++ b/Import/Importer.php @@ -72,17 +72,27 @@ public function init($file, $class, $delimiter = ',', $headerFormat = 'title') * * @return true if successful */ - public function import($fields) + public function import($fields, $fieldIndexes) { $fields = array_unique($this->caseConverter->toPascalCase($fields)); + $indexes = array_unique($this->caseConverter->toPascalCase($fieldIndexes)); + + // make keys for both arrays match + $fieldIndexes = array(); + foreach ($indexes as $k => $v) + { + $i = array_search($v, $fields); + $fieldIndexes[$i] = $v; + } while ($row = $this->reader->getRow()) { - if (($this->importCount % $this->batchSize) == 0) { - $this->addRow($row, $fields, true); - } else { - $this->addRow($row, $fields, false); - } - $this->importCount++; + ++$this->importCount; + $this->addRow($row, $fields, $fieldIndexes, (($this->importCount % $this->batchSize) == 0)); + //if (($this->importCount % $this->batchSize) == 0) { + // $this->addRow($row, $fields, true); + //} else { + // $this->addRow($row, $fields, false); + //} } // one last flush to make sure no persisted objects get left behind @@ -98,11 +108,35 @@ public function import($fields) * @param array $fields An array of the fields to import * @param boolean $andFlush Flush the ObjectManager */ - private function addRow($row, $fields, $andFlush = true) + private function addRow($row, $fields, $fieldIndexes, $andFlush = true) { - // Create new entity - $entity = new $this->class(); + // If we have indexes, try to find a record that matches and load it... + if (count($fieldIndexes) > 0) + { + //echo "
 fields = "; var_dump($fields);
+            //echo "
 fieldIndexes = "; var_dump($fieldIndexes);
+            //echo "
 row = "; var_dump($row);
+
+            $criteria = array();
+            foreach ($fieldIndexes as $k => $v)
+            {
+                $criteria[lcfirst($v)] = $row[$k];
+            }
+            //echo "
 criteria = "; var_dump($criteria);
+            $entity = $this->objectManager->getRepository($this->class)->findOneBy($criteria);
+        }
 
+        // if we don't have indexes, or we didn't get a match, create a new entity.        
+        if (!isset($entity))
+        {
+            //echo "
 entity not found
\n";
+            // Create new entity
+            $entity = new $this->class();
+        }
+        //else
+        //{
+        //    echo "
 id = ".$entity->getId() . "
\n";
+        //}
         if (in_array('Id', $fields)) {
             $key = array_search('Id', $fields);
             if ($this->metadata->hasField('legacyId')) {
@@ -114,7 +148,9 @@ private function addRow($row, $fields, $andFlush = true)
         // loop through fields and set to row value
         foreach ($fields as $k => $v) {
             if ($this->metadata->hasField(lcfirst($v))) {
-                $entity->{'set'.$fields[$k]}($row[$k]);
+                //$entity->{'set'.$fields[$k]}($row[$k]);
+                //echo '
Calling set'.$v.'('.$row[$k].')
';
+                $entity->{'set'.$v}($row[$k]);
             } else if ($this->metadata->hasAssociation(lcfirst($v))) {
                 $association = $this->metadata->associationMappings[lcfirst($v)];
                 switch ($association['type']) {
diff --git a/Resources/views/Import/mapping.html.twig b/Resources/views/Import/mapping.html.twig
index 5accd86..19adbf5 100644
--- a/Resources/views/Import/mapping.html.twig
+++ b/Resources/views/Import/mapping.html.twig
@@ -34,7 +34,6 @@
                 {{ form_row(form.delimiter) }}
                 {{ form_row(form.filename) }}
             
-            
             {{ form_widget(form._token) }}
         

@@ -42,6 +41,9 @@
              Back
         

+{% endblock %}
+{% block javascripts %}
+{{ parent() }}
     
 {% endblock %}
+
diff --git a/Util/.AbstractBase.php.swp b/Util/.AbstractBase.php.swp
deleted file mode 100644
index 5306f3082205fcda3953171319c8ccdf3050152c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 12288
zcmeI2zi$&U6o6mYDp5eNF}WpDQqlHOIut1d)Ru(-F_oz(y7=A^qs#eZ`w~zoU0`M7
zKVV>m2?@bJ0Sje-g^7U~CZ3aPlBPeHt2{?vd?!EoJ%65}yk_nGox5UXaT#b_0QmOm
z!M*R7Kf$jlfJYH_mGcWHk9CzKPnwAF$zc$5LKAem-JsLb*4ZF7ND0zNcV&WMs?vv6
z$F3V1T$H*U$lMrAUC==aF+np=qXccR)-gKRu$}eZd$8$L609|?Gof_rp~c0lmHnp>
zUS<~18jf|umLtO(FR;rf?YCj
zrm!H2ximHxuRO5<~1E%Lumk%a5Hxmr-~OrWTFNfml!>;v
zL2*Nri^qB|B+HwLk|NGi>6A`|MYqupt?mcTiEa7~N8`|@5hkjw92&pv;N{qjZ|{d!
zhrvf|K1}12#=cyv7RP9XzF~W=pTaMzSX7m57_>S!ay-uO?xV&}*=R&a-QBr#)guZx
zkc*BM>kQLsISXBDUXpwNB@<)ejvjawb*@Ht?~cF7+bd`0Auii}FUZQYjU4
zwKAwUGuoka7>P2b!was=R*EYat`9E9?=gRPz3;US+YlCQ??NV_wPLgj9v}_dXtPkF
W@E!A8gO2$v-!WeYSiR*N<^KWq0}-|W

diff --git a/Util/.Writer.php.swp b/Util/.Writer.php.swp
deleted file mode 100644
index 813fb72e55732476bc3b44ef00064ab38153174b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 12288
zcmeI2L2DE-6vtn^d8ifzK}C$Zva`@`sy!&wR(6$w2NA^TrBbppnQn9@8IsIQmm=cH
zizg3$2S0$Q7yTUi75oHx@?~djSB3Vf^gZ||o6XCAUVgV>-eB?JoxAK>cL^}&0M5O6
zeDC(zPw?$1z>`GQ%7=~pqpz}TG>|ggKMi;#EU)XDS3~2xFSlm0N~Qa3+&{H^~1@6ZcPM;01+SpM1Tko0U|&I4k-a&%)oQJ`;p20Pfg~T
zKjz7jCL%xthyW2F0z`la5CI}U1c(3;AOb|-KO`U$fVZ>w-TxRCkKg})z5o9>4)7iI
z74-%69`z3O8ubd5qt;MoP;;oyc>W{m1L`H}1?mBcqb{S)qrx*zqbP^~5g-CYfCvx)
zB0vO)01+SpMBrZ{@C*YOMl=5YE38_6sNBM89eS_tn>uP$A}eKQ8Nz0W)L1FvA&XkV
zS}|h6vDQ;LVk@g`lFN1=BsODF4l)(9wA8UzMzgW;nl*VZoQ@ig#;OD3b9BlJZo7~tj`?MP8#hC{{7)LC)UE&k

diff --git a/Util/FieldRetriever.php b/Util/FieldRetriever.php
index 486b92e..110505d 100644
--- a/Util/FieldRetriever.php
+++ b/Util/FieldRetriever.php
@@ -5,6 +5,7 @@
 use Doctrine\Common\Annotations\AnnotationReader;
 
 use Avro\CsvBundle\Annotation\ImportExclude;
+use Avro\CsvBundle\Annotation\ImportIndex;
 use Avro\CaseBundle\Util\CaseConverter;
 
 /**
@@ -35,7 +36,7 @@ public function __construct($annotationReader, CaseConverter $caseConverter)
      * @param string  $format    The desired field case format
      * @param boolean $copyToKey Copy the field values to their respective key
      *
-     * @return array $fields
+     * @return array(array $fields, array $indexFields)
      */
     public function getFields($class, $format = 'title', $copyToKey = false)
     {
@@ -43,12 +44,16 @@ public function getFields($class, $format = 'title', $copyToKey = false)
         $properties = $reflectionClass->getProperties();
 
         $fields = array();
+        $fieldIndexes = array();
         foreach ($properties as $property) {
             $addField = true;
             foreach ($this->annotationReader->getPropertyAnnotations($property) as $annotation) {
                 if ($annotation instanceof ImportExclude) {
                     $addField = false;
                 }
+                if ($annotation instanceof ImportIndex) {
+                    $fieldIndexes[] = $this->caseConverter->convert($property->getName(), $format);
+                }
             }
 
             if ($addField) {
@@ -58,8 +63,9 @@ public function getFields($class, $format = 'title', $copyToKey = false)
 
         if ($copyToKey) {
             $fields = array_combine($fields, $fields);
+            $fieldIndexes = array_combine($fieldIndexes, $fieldIndexes);
         }
 
-        return $fields;
+        return array($fields, $fieldIndexes);
     }
 }
diff --git a/Util/Reader.php b/Util/Reader.php
index 6584e72..d30d7bd 100644
--- a/Util/Reader.php
+++ b/Util/Reader.php
@@ -15,6 +15,7 @@ class Reader
     protected $enclosure;
     protected $line;
     protected $headers;
+    protected $lineLength;
 
     /**
      * Open a CSV file
@@ -25,12 +26,13 @@ class Reader
      * @param string  $enclosure  The enclosure
      * @param boolean $hasHeaders Does the CSV have any headers?
      */
-    public function open($file, $delimiter = ',', $mode = 'r+', $enclosure = '"', $hasHeaders = true)
+    public function open($file, $delimiter = ',', $mode = 'r+', $enclosure = '"', $hasHeaders = true, $lineLength = 0)
     {
         $this->handle = fopen($file, $mode);
         $this->delimiter = $delimiter;
         $this->enclosure = $enclosure;
         $this->line = 0;
+        $this->lineLength = $lineLength;
 
         if ($hasHeaders) {
             $this->headers = $this->getRow();
@@ -44,13 +46,15 @@ public function open($file, $delimiter = ',', $mode = 'r+', $enclosure = '"', $h
      */
     public function getRow()
     {
-        if (($row = fgetcsv($this->handle, 1000, $this->delimiter, $this->enclosure)) !== false) {
+        while (($row = fgetcsv($this->handle, $this->lineLength, $this->delimiter, $this->enclosure)) !== false) {
             $this->line++;
+            // a blank line returns array of one null. if found, skip it.
+            if ((count($row) == 1) && ($row[0] == NULL))
+              continue;
 
             return $row;
-        } else {
-            return false;
         }
+        return false;
     }
 
     /**

From b01f9d666f7466ecf3cf35afcd51f19a7a1416d9 Mon Sep 17 00:00:00 2001
From: jp31415926 
Date: Sat, 5 Jan 2013 02:09:35 -0600
Subject: [PATCH 2/2] Clean up

---
 Import/Importer.php | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/Import/Importer.php b/Import/Importer.php
index cdf3548..d1cd626 100644
--- a/Import/Importer.php
+++ b/Import/Importer.php
@@ -88,11 +88,6 @@ public function import($fields, $fieldIndexes)
         while ($row = $this->reader->getRow()) {
             ++$this->importCount;
             $this->addRow($row, $fields, $fieldIndexes, (($this->importCount % $this->batchSize) == 0));
-            //if (($this->importCount % $this->batchSize) == 0) {
-            //    $this->addRow($row, $fields, true);
-            //} else {
-            //    $this->addRow($row, $fields, false);
-            //}
         }
 
         // one last flush to make sure no persisted objects get left behind
@@ -148,8 +143,6 @@ private function addRow($row, $fields, $fieldIndexes, $andFlush = true)
         // loop through fields and set to row value
         foreach ($fields as $k => $v) {
             if ($this->metadata->hasField(lcfirst($v))) {
-                //$entity->{'set'.$fields[$k]}($row[$k]);
-                //echo '
Calling set'.$v.'('.$row[$k].')
';
                 $entity->{'set'.$v}($row[$k]);
             } else if ($this->metadata->hasAssociation(lcfirst($v))) {
                 $association = $this->metadata->associationMappings[lcfirst($v)];