Remaster (#38)
* setup new 'statistiques' module * added 'graphos' package and created first test graph * put graphos in requirements, deleted local folder * added "load_csv" management command ! * added update of premiere_rencontre field in 'load_csv' management command * added missing urls.py file * added 'merge' action and view * added 'info_completed' ratio * linked sujets:merge views inside suivi:details * added link to maraudes:details in notes table headers, if any * Major reorganisation, moved 'suivi' and 'sujets' to 'notes', cleanup in 'maraudes', dropping 'website' mixins (mostly useless) * small cleanup * worked on Maraude and Sujet lists * corrected missing line in notes.__init__ * restored 'details' view for maraudes and sujets insie 'notes' module * worked on 'notes': added navigation between maraude's compte-rendu, right content in details, header to list tables * changed queryset for CompteRenduDetailsView to all notes of same date, minor layout changes * added right content to 'details-sujet', created 'statistiques' view and update templates * restored 'statistiques' ajax view in 'details-sujet', fixed 'merge_two' util function * added auto-creation of FicheStatistique (plus some tests), pagination for notes in 'details-sujet' * added error-prone cases in paginator * fixed non-working modals, added titles * added UpdateStatistiques capacity in CompteRenduCreate view * fixed missing AjaxTemplateMixin for CreateSujetView, worked on compte-rendu creation scripts * fixed MaraudeManager.all_of() for common Maraudeurs, added color hints in planning * re-instated statistiques module link and first test page * added FinalizeView to send a mail before finalizing compte-rendu * Added PieChart view for FicheStatistique fields * small style updates, added 'age' and 'genre' fields from sujets in statistiques.PieChartView * worked on statistiques, fixed small issues in 'notes' list views * small theme change * removed some dead code * fixed notes.tests, fixed statistiques.info_completed display, added filter in SujetLisView * added some tests * added customised admin templates * added authenticate in CustomAuthenticatationBackend, more verbose login thanks to messages * added django-nose for test coverage * Corrected raising exception on first migration On first migration, qs.exists() would previously be called and raising an Exception, sot he migrations would fail. * Better try block * cleaned up custom settings.py, added some overrides of django base_settings * corrected bad dictionnary key
This commit is contained in:
artus40
GitHub
216
maraudes/management/commands/load_csv.py
Normal file
216
maraudes/management/commands/load_csv.py
Normal file
@@ -0,0 +1,216 @@
|
||||
import datetime
|
||||
import csv
|
||||
|
||||
def parse_rows(data_file):
|
||||
|
||||
reader = csv.DictReader(data_file, delimiter=";")
|
||||
|
||||
for i, row in enumerate(reader):
|
||||
date = datetime.datetime.strptime(row['DATE'] + ".2016", "%d.%m.%Y").date()
|
||||
lieu = row['LIEUX']
|
||||
nom, prenom = row['NOM'], row['PRENOM']
|
||||
prem_rencontre = True if row['1 ER CONTACT'].lower() == "oui" else False
|
||||
|
||||
yield i, date, lieu, nom, prenom, prem_rencontre
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
||||
help = "Load data for rencontre from csv files"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('file', help="path to the files to load", nargs="+", type=str)
|
||||
parser.add_argument('--commit', help="commit changes to the database",
|
||||
action="store_true", dest="commit", default=False)
|
||||
parser.add_argument('--check', action='store_true', dest='check', default=False,
|
||||
help="Check that all lines from file are written into database")
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
if not hasattr(self, '_cache'):
|
||||
self._cache = {'maraude': {}, 'lieu': {}, 'sujet': {}, 'rencontre': [], 'observation': []}
|
||||
return self._cache
|
||||
|
||||
def new_object(self, model, data, cache_key=None):
|
||||
""" Create new object, add it to cache (in dict if cache_key is given, in list otherwise).
|
||||
Save it only if --commit option is given
|
||||
"""
|
||||
obj = model(**data)
|
||||
msg = "[%i]+ Created %s " % (self.cur_line, obj)
|
||||
if self._commit:
|
||||
obj.save()
|
||||
msg += " successfully saved to db"
|
||||
if cache_key:
|
||||
self.cache[model.__qualname__.lower()][cache_key] = obj
|
||||
msg += " and added to cache."
|
||||
self.stdout.write(self.style.SUCCESS(msg))
|
||||
return obj
|
||||
|
||||
@property
|
||||
def referent_maraude(self):
|
||||
if not hasattr(self, '_referent'):
|
||||
from utilisateurs.models import Maraudeur
|
||||
self._referent = Maraudeur.objects.get_referent()
|
||||
return self._referent
|
||||
|
||||
def find_maraude(self, date):
|
||||
from maraudes.models import Maraude
|
||||
try: # First, try to retrieve from database
|
||||
obj = Maraude.objects.get(date=date)
|
||||
except Maraude.DoesNotExist:
|
||||
# Try to retrieve from cache
|
||||
try:
|
||||
obj = self.cache['maraude'][date]
|
||||
except KeyError:
|
||||
# Create a new object and put it into cache
|
||||
obj = self.new_object(
|
||||
Maraude,
|
||||
{'date':date, 'referent':self.referent_maraude, 'binome':self.referent_maraude},
|
||||
cache_key=date)
|
||||
return obj
|
||||
|
||||
def find_sujet(self, nom, prenom):
|
||||
from sujets.models import Sujet
|
||||
from watson import search
|
||||
|
||||
search_text = "%s %s" % (nom, prenom)
|
||||
sujet = self.cache['sujet'].get(search_text, None)
|
||||
|
||||
while not sujet:
|
||||
create = False #Set to True if creation is needed at and of loop
|
||||
self.stdout.write(self.style.WARNING("In line %i, searching : %s. " % (self.cur_line, search_text)), ending='')
|
||||
results = search.filter(Sujet, search_text)
|
||||
|
||||
if results.count() == 1: # Always ask to review result a first time
|
||||
sujet = results[0]
|
||||
self.stdout.write(self.style.SUCCESS("Found %s '%s' %s" % (sujet.nom, sujet.surnom, sujet.prenom)))
|
||||
action = input("Confirm ? (y/n/type new search)> ")
|
||||
if action == "n":
|
||||
sujet = None
|
||||
search_text = "%s %s" % (nom, prenom)
|
||||
elif action == "y":
|
||||
continue
|
||||
else: # In case the result is no good at all !
|
||||
sujet = None
|
||||
search_text = action
|
||||
|
||||
elif results.count() > 1: # Ask to find the appropriate result
|
||||
self.stdout.write(self.style.WARNING("Multiple results for %s" % search_text))
|
||||
for i, result in enumerate(results):
|
||||
self.stdout.write("%i. %s '%s' %s" % (i, result.nom, result.surnom, result.prenom))
|
||||
choice = input("Choose the right number - Type new search - C to create '%s %s': " % (nom, prenom))
|
||||
if choice == "C":
|
||||
create = True
|
||||
else:
|
||||
try: sujet = results[int(choice)]
|
||||
except (IndexError, ValueError):
|
||||
search_text = str(choice) #New search
|
||||
continue
|
||||
|
||||
else: # No results, try with name only, or ask for new search
|
||||
if search_text == "%s %s" % (nom, prenom):
|
||||
# Search only with nom
|
||||
self.stdout.write(self.style.WARNING("Nothing, trying name only..."), ending='')
|
||||
search_text = nom if nom else prenom
|
||||
continue
|
||||
else:
|
||||
self.stdout.write(self.style.ERROR("No result !"))
|
||||
action = input("New search or C to create '%s %s': " % (nom, prenom))
|
||||
if action == "C":
|
||||
create = True
|
||||
else:
|
||||
search_text = str(action)
|
||||
|
||||
if create:
|
||||
sujet = self.new_object(
|
||||
Sujet,
|
||||
{'nom':nom, 'prenom':prenom}
|
||||
)
|
||||
self.stdout.write('Created, %s' % sujet)
|
||||
# Always store sujet in cache because it may or may not be updated, safer to save in all cases.
|
||||
self.cache['sujet']["%s %s" % (nom, prenom)] = sujet
|
||||
return sujet
|
||||
|
||||
def find_lieu(self, nom):
|
||||
from maraudes.models import Lieu
|
||||
|
||||
try:
|
||||
lieu = Lieu.objects.get(nom=nom)
|
||||
except Lieu.DoesNotExist:
|
||||
lieu = self.cache['lieu'].get(nom, None)
|
||||
while not lieu:
|
||||
self.stdout.write(self.style.WARNING("At line %i, le lieu '%s' n'a pas été trouvé" % (self.cur_line, nom)))
|
||||
action = input('%s (Créer/Sélectionner)> ' % nom)
|
||||
if action == "C":
|
||||
lieu = self.new_object(Lieu, {'nom': nom}, cache_key=nom)
|
||||
elif action == "S":
|
||||
choices = {l.pk:l.nom for l in Lieu.objects.all()}
|
||||
for key, name in choices.items():
|
||||
self.stdout.write("%i. %s" % (key, name))
|
||||
while not lieu:
|
||||
chosen_key = input('Choose a number: ')
|
||||
try:
|
||||
lieu = Lieu.objects.get(pk=chosen_key)
|
||||
confirm = input("Associer %s à %s ? (o/n)> " % (nom, lieu.nom))
|
||||
if confirm == "n":
|
||||
lieu = None
|
||||
else:
|
||||
self.cache['lieu'][nom] = lieu
|
||||
|
||||
except (Lieu.DoesNotExist, ValueError):
|
||||
lieu = None
|
||||
else:
|
||||
continue
|
||||
|
||||
return lieu
|
||||
|
||||
def add_rencontre(self, maraude, sujet, lieu):
|
||||
from maraudes.models import Rencontre
|
||||
from maraudes.notes import Observation
|
||||
|
||||
rencontre = self.new_object(Rencontre,
|
||||
{'maraude':maraude, 'lieu':lieu, 'heure_debut':datetime.time(20, 0),
|
||||
'duree':15})
|
||||
observation = self.new_object(Observation, {'rencontre':rencontre,
|
||||
'sujet':sujet,
|
||||
'text':"Chargé depuis '%s'" % self._file.name})
|
||||
self.cache['rencontre'].append(rencontre)
|
||||
self.cache['observation'].append(observation)
|
||||
|
||||
def handle(self, **options):
|
||||
""" Parsing all given files, look for existing objects and create new Rencontre
|
||||
and Observation objects. Ask for help finding related object, creating new ones
|
||||
if needed. All creation/updates are stored in cache and commited only after
|
||||
user confirmation
|
||||
"""
|
||||
|
||||
self._commit = options.get('commit', False)
|
||||
|
||||
for file_path in options['file']:
|
||||
with open(file_path, 'r') as data_file:
|
||||
self.stdout.write("Working with '%s'" % data_file.name)
|
||||
self._file = data_file
|
||||
for line, date, lieu, nom, prenom, prems in parse_rows(data_file):
|
||||
self.cur_line = line
|
||||
maraude = self.find_maraude(date)
|
||||
lieu = self.find_lieu(lieu)
|
||||
sujet = self.find_sujet(nom, prenom)
|
||||
assert sujet is not None
|
||||
assert lieu is not None
|
||||
assert maraude is not None
|
||||
if prems and self._commit:
|
||||
sujet.premiere_rencontre = date
|
||||
sujet.save()
|
||||
self.stdout.write(self.style.SUCCESS("[%i]* Updated premiere_rencontre on %s" % (self.cur_line, sujet)))
|
||||
|
||||
self.add_rencontre(maraude, sujet, lieu)
|
||||
|
||||
#Summary
|
||||
self.stdout.write(" ## %s : %i lines ##" % (data_file.name, self.cur_line))
|
||||
self.stdout.write("Trouvé %s nouvelles observations" % len(self.cache['observation']))
|
||||
self.stdout.write("Nécessite l'ajout/modification de : \n- %i maraudes\n- %i lieux\n- %i sujets" %
|
||||
(len(self.cache['maraude']), len(self.cache['lieu']), len(self.cache['sujet'])))
|
||||
|
||||
view = input('Voulez-vous voir la liste des changements ? (o/n)> ')
|
||||
if view == "o": self.stdout.write(" ## Changements ## \n%s" % self.cache)
|
||||
Reference in New Issue
Block a user