Cadoles
/
ninegate
20
0
Fork 0
Code Issues 29 Pull Requests Projects 1 Releases Wiki Activity

optimisation synchro LDAP to SQL

This commit is contained in:
afornerot 2019-08-23 17:01:04 +02:00
parent 60cce2f317
commit 25089400ce
2 changed files with 64 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/ninegate-1.0/src/Cadoles/CoreBundle/Command/InitDataCommand.php
View File

@ -113,16 +113,23 @@ class InitDataCommand extends ContainerAwareCommand
->from('CadolesCoreBundle:User','user')
->where($qb->expr()->not($qb->expr()->exists($sub->getDQL())))
->setParameter("groupid",$group->getId());
$datas=$qb->getQuery()->getResult();
$datas=$qb->getQuery()->getArrayResult();
$nbusers=0;
foreach($datas as $data) {
$output->writeln(" > Ratachement ".$data->getId()." ".$data->getUsername());
$nbusers++;
$user=$em->getRepository('CadolesCoreBundle:User')->find($data["id"]);
$group=$em->getRepository('CadolesCoreBundle:Group')->findOneBy(array('fgall'=>true));
$output->writeln(" $nbusers sur ".count($datas)." > Ratachement ".$user->getId()." ".$user->getUsername());
$key = Uuid::uuid4();
$usergroup=new UserGroup();
$usergroup->setUser($data);
$usergroup->setUser($user);
$usergroup->setGroup($group);
$usergroup->setKeyvalue($key);
$em->persist($usergroup);
$em->flush();
$em->clear();
//$user=$em->getRepository('CadolesCoreBundle:User')->find($data["id"]);
}
// On s'assure si masteridentity est à LDAP qu'au minimum un niveau01 possède un filtre LDAP

76
src/ninegate-1.0/src/Cadoles/CoreBundle/Command/SynchroCommand.php
View File

@ -24,10 +24,10 @@ class SynchroCommand extends Command
{
private $container;
private $em;
private $em2;
private $output;
private $filesystem;
private $rootlog;
private $cptcommit;
protected function configure()
{
@ -141,21 +141,29 @@ class SynchroCommand extends Command
$this->writeln('== USERS ============================================');
// On stocke tout les email déjà existant
$users=$this->em->getRepository('CadolesCoreBundle:User')->findAll();
$this->writeln('== Stocker les emails utilisateurs existants');
$users=$this->em->createQueryBuilder()->select('table.email')->from('CadolesCoreBundle:User','table')->getQuery()->getArrayResult();
foreach($users as $user) {
array_push($ldapmails,$user->getEmail());
}
array_push($ldapmails,$user["email"]);
}
// On parcours les niveaux 01 pour connaitre les filtres ldap associé
$nbusers=0;
$this->cptcommit=0;
$datas=$this->em->createQueryBuilder()->select('table')->from('CadolesCoreBundle:Niveau01','table')->where('table.ldapfilter IS NOT NULL')->getQuery()->getResult();
$datas=$this->em->createQueryBuilder()->select('table')->from('CadolesCoreBundle:Niveau01','table')->where('table.ldapfilter IS NOT NULL')->getQuery()->getArrayResult();
$tberrors=[];
foreach($datas as $data) {
// On execute le filtre d'appartenance à ce niveau
$results = $ldap->search($data->getLdapfilter(), $fieldstoread, $ldap_basedn);
$this->writeln("== Récupération des utilisateurs de l'annuaire");
$niveau01=$this->em->getRepository('CadolesCoreBundle:Niveau01')->find($data["id"]);
$results = $ldap->search($niveau01->getLdapfilter(), $fieldstoread, $ldap_basedn);
$nbuserstotal=count($results);
// Pour chaque utilisateur ldap
$this->writeln('== Traitement des utilisateurs du niveau = '.$niveau01->getLabel(). " = ".$niveau01->getLdapfilter());
foreach($results as $result) {
// Niveau 01 en cours à récupérer car on clear à em à chaque $result
$niveau01=$this->em->getRepository('CadolesCoreBundle:Niveau01')->find($data["id"]);
// Compteur de users
$nbusers++;
@ -181,14 +189,14 @@ class SynchroCommand extends Command
$user=$this->em->getRepository('CadolesCoreBundle:User')->findOneBy(array('username' => $result[$ldap_username]));
if(!$user) {
if(empty($result[$ldap_email]))
$this->writelnred(" - Création dans Bundle impossible >> ".$result[$ldap_username]." sans email");
array_push($tberrors," - Création dans Bundle impossible >> ".$result[$ldap_username]." sans email");
else {
if(in_array($result[$ldap_email],$ldapmails))
$this->writelnred(" - Création dans Bundle impossible >> ".$result[$ldap_username]." un autre utilisateur a déjà ce mail = ".$result[$ldap_email]);
array_push($tberrors," - Création dans Bundle impossible >> ".$result[$ldap_username]." un autre utilisateur a déjà ce mail = ".$result[$ldap_email]);
else {
array_push($ldapmails,$result[$ldap_email]);
$this->writeln(" - Création dans Bundle >> ".$result[$ldap_username]);
if(!$simulate) $this->addUser($data,$result[$ldap_username],$result[$ldap_firstname],$result[$ldap_lastname],$result[$ldap_email],$ldap_usersadmin);
if(!$simulate) $this->addUser($niveau01,$result[$ldap_username],$result[$ldap_firstname],$result[$ldap_lastname],$result[$ldap_email],$ldap_usersadmin);
}
}
}
@ -200,7 +208,7 @@ class SynchroCommand extends Command
if($user->getLastname()!=$result[$ldap_lastname]||$user->getFirstname()!=$result[$ldap_firstname]||$user->getEmail()!=$result[$ldap_email]||$toadmin) {
$usermail=$this->em->getRepository('CadolesCoreBundle:User')->findOneBy(array('email' => $result[$ldap_email]));
if($usermail!=$user) {
$this->writelnred(" - Modification dans Bundle impossible >> ".$result[$ldap_username]." un autre utilisateur a déjà ce mail = ".$result[$ldap_email]);
array_push($tberrors," - Modification dans Bundle impossible >> ".$result[$ldap_username]." un autre utilisateur a déjà ce mail = ".$result[$ldap_email]);
}
else {
$this->writeln(" - Modification dans Bundle >> ".$result[$ldap_username]);
@ -209,15 +217,21 @@ class SynchroCommand extends Command
}
}
if(!$simulate&&$this->cptcommit>0&&($this->cptcommit%100==0)) {
$this->writelnred(" == COMMIT ==");
$this->em->flush();
if(!$simulate) {
$this->em->clear();
}
if($nbusers%1000==0) $this->writeln(" == Nombre d'utilisateurs traités = $nbusers sur $nbuserstotal ==");
}
}
if(!$simulate) {
$this->writelnred(" == COMMIT ==");
$this->writeln(" == Nombre d'utilisateurs traités = $nbusers sur $nbuserstotal ==");
$this->em->flush();
$this->em->clear();
}
foreach($tberrors as $error) {
$this->writelnred(" == ERROR == $error");
}
$this->writeln('');
@ -282,6 +296,7 @@ class SynchroCommand extends Command
}
}
}
$this->writeln('');
$this->writeln('=====================================================');
@ -292,20 +307,37 @@ class SynchroCommand extends Command
$this->writeln('== USERS ============================================');
// Pour chaque utilisateur de la base
$users=$this->em->getRepository('CadolesCoreBundle:User')->findAll();
foreach($users as $user) {
//$users=$this->em->getRepository('CadolesCoreBundle:User')->findAll();
$datas=$this->em->createQueryBuilder()->select('table.id,table.username')->from('CadolesCoreBundle:User','table')->getQuery()->getArrayResult();
$nbusers=0;
// tentative d'optimisation
$flipped = array_flip($ldapusers);
foreach($datas as $data) {
$nbusers++;
// Si l'utilisateur n'est pas dans la liste des users ldap : on le supprime
if(!in_array($user->getUsername(),$ldapusers)) {
if(!isset($flipped[$data["username"]])) {
$user=$this->em->getRepository('CadolesCoreBundle:User')->find($data["id"]);
if($user->getUsername()=="admin")
$this->writeln(" - Ne jamais supprimer >> ".$user->getUsername());
else {
$this->writeln(" - Suppression dans Bundle >> ".$user->getUsername());
if(!$simulate) {
$this->em->remove($user);
$this->em->flush();
$this->em->flush();
}
}
}
else {
//on peut unset les recherches suivantes seront plus rapide
unset($flipped[$user->getUsername()]);
}
$this->em->clear();
if($nbusers%1000==0) $this->writelnred(" == Nombre d'utilisateurs traités = $nbusers==");
}
}
}
@ -673,8 +705,8 @@ class SynchroCommand extends Command
else
$user->setRole("ROLE_USER");
$this->cptcommit++;
$this->em->persist($user);
$this->em->flush();
}
protected function modUser($user,$username,$firstname,$lastname,$email,$usersadmin) {
@ -685,8 +717,8 @@ class SynchroCommand extends Command
if(in_array($username,$usersadmin))
$user->setRole("ROLE_ADMIN");
$this->cptcommit++;
$this->em->persist($user);
$this->em->flush();
}
}