Source code

001package org.nasdanika.models.gitlab.util;
002
003import java.util.ArrayList;
004import java.util.Collection;
005import java.util.Collections;
006import java.util.HashMap;
007import java.util.List;
008import java.util.Map;
009import java.util.Objects;
010import java.util.concurrent.CompletableFuture;
011import java.util.concurrent.Executor;
012import java.util.function.Function;
013import java.util.logging.Handler;
014import java.util.logging.Level;
015import java.util.logging.Logger;
016
017import org.eclipse.emf.common.util.EList;
018import org.eclipse.emf.common.util.EMap;
019import org.gitlab4j.api.Constants.AutoDevopsDeployStrategy;
020import org.gitlab4j.api.Constants.BuildGitStrategy;
021import org.gitlab4j.api.Constants.SquashOption;
022import org.gitlab4j.api.GitLabApi;
023import org.gitlab4j.api.GitLabApiException;
024import org.gitlab4j.api.GroupApi;
025import org.gitlab4j.api.Pager;
026import org.gitlab4j.api.ProjectApi;
027import org.gitlab4j.api.ProjectLicense;
028import org.gitlab4j.api.RepositoryApi;
029import org.gitlab4j.api.RepositoryFileApi;
030import org.gitlab4j.api.models.AbstractUser;
031import org.gitlab4j.api.models.Branch;
032import org.gitlab4j.api.models.Commit;
033import org.gitlab4j.api.models.Contributor;
034import org.gitlab4j.api.models.CustomAttribute;
035import org.gitlab4j.api.models.Group;
036import org.gitlab4j.api.models.Group.Statistics;
037import org.gitlab4j.api.models.GroupFilter;
038import org.gitlab4j.api.models.Member;
039import org.gitlab4j.api.models.Owner;
040import org.gitlab4j.api.models.Permissions;
041import org.gitlab4j.api.models.Project;
042import org.gitlab4j.api.models.ProjectAccess;
043import org.gitlab4j.api.models.ProjectSharedGroup;
044import org.gitlab4j.api.models.ProjectStatistics;
045import org.gitlab4j.api.models.RepositoryFile;
046import org.gitlab4j.api.models.TreeItem;
047import org.gitlab4j.api.models.Visibility;
048import org.nasdanika.common.ProgressMonitor;
049import org.nasdanika.common.Status;
050import org.nasdanika.models.gitlab.GitLab;
051import org.nasdanika.models.gitlab.GitLabFactory;
052import org.nasdanika.models.gitlab.MergeMethod;
053
054/**
055 * Loads data into the model using {@link GitLabApi}.
056 * @deprecated Migrate to Loader
057 */
058@Deprecated
059public class LoaderOld implements AutoCloseable {
060        
061        private static final String ROOT_PATH = "/";
062        private GitLabApi gitLabApi;
063        private GitLabFactory factory = GitLabFactory.eINSTANCE;
064        private int groupsPageSize = 10;
065        
066        public int getGroupsPageSize() {
067                return groupsPageSize;
068        }
069        
070        /**
071         * Page size for retrieving groups.
072         * @param groupsPageSize
073         */
074        public void setGroupsPageSize(int groupsPageSize) {
075                this.groupsPageSize = groupsPageSize;
076        }
077        
078        // Caller thread executor
079        private Executor executor = r -> r.run();
080        
081        public Executor getExecutor() {
082                return executor;
083        }
084        
085        /**
086         * @param executor To execute loading in parallel
087         */
088        public void setExecutor(Executor executor) {
089                this.executor = executor;
090        }
091        
092        /**
093         * Access to the API for configuration. 
094         * @return
095         */
096        public GitLabApi getGitLabApi() {
097                return gitLabApi;
098        }
099        
100        public LoaderOld(String hostUrl, String accessToken) {
101                this(new GitLabApi(hostUrl, accessToken));
102        }       
103        
104        public LoaderOld(GitLabApi gitLabApi) {
105                this(gitLabApi, new ThrottlingHandler());
106        }
107        
108        /**
109         * @param clientRateLimitWindow Client rate window in milliseconds. Client rate limit is enforced if this value and clientRateLimit are positive.
110         * @param clientRateLimit Client rate limit per rate window. Client rate limit is enforced if this value and clientRateLimitWindow are positive.
111         */     
112        public LoaderOld(
113                        String hostUrl, 
114                        String accessToken, 
115                        long clientRateLimitWindow,
116                        int clientRateLimit) {
117                this(new GitLabApi(hostUrl, accessToken), clientRateLimitWindow, clientRateLimit);
118        }       
119        
120        /**
121         * @param clientRateLimitWindow Client rate window in milliseconds. Client rate limit is enforced if this value and clientRateLimit are positive.
122         * @param clientRateLimit Client rate limit per rate window. Client rate limit is enforced if this value and clientRateLimitWindow are positive.
123         */     
124        public LoaderOld(
125                        GitLabApi gitLabApi,
126                        long clientRateLimitWindow,
127                        int clientRateLimit) {
128                this(gitLabApi, new ThrottlingHandler(clientRateLimitWindow, clientRateLimit));
129        }       
130        
131        public LoaderOld(GitLabApi gitLabApi, Handler throttlingHandler) {
132                if (throttlingHandler != null) {                        
133                        Level level = Level.FINE;
134                        throttlingHandler.setLevel(level);
135                        Logger logger = Logger.getLogger(GitLabApi.class.getName());
136                        Level loggerLevel = logger.getLevel();
137                        if (loggerLevel == null || loggerLevel.intValue() > level.intValue()) {
138                                logger.setLevel(level);
139                        }
140                        logger.addHandler(throttlingHandler);
141                        gitLabApi.enableRequestResponseLogging();
142                }
143                
144                this.gitLabApi = gitLabApi;
145        }
146        
147        public GitLabFactory getFactory() {
148                return factory;
149        }
150        
151        public void setFactory(GitLabFactory factory) {
152                this.factory = factory;
153        }
154        
155        /**
156         * Loads groups, their projects, sub-groups, members and other related objects.
157         * @param progressMonitor
158         * @return Populated {@link GitLab} instance.
159         * @throws GitLabApiException 
160         */
161        public GitLab loadGitLabGroups(ProgressMonitor progressMonitor) throws GitLabApiException {
162                GitLab ret = getFactory().createGitLab();               
163                ret.getGroups().addAll(loadGroups(progressMonitor));                            
164                return ret;
165        }
166        
167        protected org.nasdanika.models.gitlab.ProjectLicense loadProjectLicense(ProjectLicense apiLicense, ProgressMonitor progressMonitor) {
168                org.nasdanika.models.gitlab.ProjectLicense modelLicense = getFactory().createProjectLicense();
169                modelLicense.setHtmlUrl(apiLicense.getHtmlUrl());
170                modelLicense.setKey(apiLicense.getKey());
171                modelLicense.setName(apiLicense.getName());
172                modelLicense.setNickname(apiLicense.getNickname());
173                modelLicense.setSourceUrl(apiLicense.getSourceUrl());
174                return modelLicense;
175        }
176
177        protected void populateAbstractUser(AbstractUser<?> apiUser, org.nasdanika.models.gitlab.AbstractUser user) {
178                user.setAvatarUrl(apiUser.getAvatarUrl());
179                user.setCreatedAt(apiUser.getCreatedAt());
180                user.setEMail(apiUser.getEmail());
181                user.setId(apiUser.getId());
182                user.setName(apiUser.getName());
183                user.setState(apiUser.getState());
184                user.setUserName(apiUser.getUsername());
185                user.setWebUrl(apiUser.getWebUrl());
186        }
187        
188        /**
189         * This implementation returns a new instance of {@link GroupFilter}.
190         * Override to customize.
191         * @return
192         */
193        protected GroupFilter getGroupFilter() {
194                return new GroupFilter();
195        }
196
197        /**
198         * @param progressMonitor
199         * @return Top-level (root) groups with sub-groups mounted under them
200         * @throws GitLabApiException
201         */
202        public List<org.nasdanika.models.gitlab.Group> loadGroups(ProgressMonitor progressMonitor) throws GitLabApiException {
203                Map<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupMap = Collections.synchronizedMap(new HashMap<>());
204                Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider = id -> groupMap.computeIfAbsent(id, _id -> new CompletableFuture<>());
205                
206                Map<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectMap = Collections.synchronizedMap(new HashMap<>());
207                Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider = id -> projectMap.computeIfAbsent(id, _id -> new CompletableFuture<>());
208
209                List<org.nasdanika.models.gitlab.Group> rootGroups = Collections.synchronizedList(new ArrayList<>()); 
210                List<org.nasdanika.models.gitlab.Group> childGroups = Collections.synchronizedList(new ArrayList<>()); 
211                
212                Collection<CompletableFuture<Map.Entry<Group, org.nasdanika.models.gitlab.Group>>> groupCompletableFutures = Collections.synchronizedCollection(new ArrayList<>()); // Synchronizing just in case
213                try (ProgressMonitor groupsMonitor = progressMonitor.split("Loading groups", 1)) {
214                        GroupApi groupApi = gitLabApi.getGroupApi();
215                        Pager<Group> groupPager = groupApi.getGroups(getGroupFilter(), getGroupsPageSize());
216                        int pageNum = 0;
217                        while (groupPager.hasNext()) {
218                                ++pageNum;
219                                double monitorSize = 1.0/Math.pow(2.0, pageNum); // Unknown number of pages, dividing each next by 2. I.e. 1/2 for the first page, 1/4 for the second, ...
220                                try (ProgressMonitor groupPageMonitor = groupsMonitor.split("Group page " + pageNum, monitorSize)) {
221                                        List<Group> groups = groupPager.next();
222                                        try (ProgressMonitor scaledGroupsMonitor = groupPageMonitor.scale(groups.size() + 1)) {
223                                                scaledGroupsMonitor.worked(Status.INFO, 1, "Retrieved " + groups.size() + " groups");
224                                                for (org.gitlab4j.api.models.Group group: groups) {
225                                                        try (ProgressMonitor groupMonitor = scaledGroupsMonitor.split("Loading group " + group.getName() + " " + group.getId(), 1, group)) {
226                                                                CompletableFuture<Map.Entry<Group,org.nasdanika.models.gitlab.Group>> modelGroupCompletableFuture = new CompletableFuture<>();
227                                                                groupCompletableFutures.add(modelGroupCompletableFuture);
228                                                                executor.execute(() -> { 
229                                                                        org.nasdanika.models.gitlab.Group modelGroup = loadGroup(
230                                                                                        group, 
231                                                                                        groupApi, 
232                                                                                        groupProvider,  
233                                                                                        projectProvider,
234                                                                                        groupMonitor);
235                                                                        
236                                                                        modelGroupCompletableFuture.complete(Map.entry(group, modelGroup));
237                                                                });
238                                                        }
239                                                }
240                                        }
241                                }
242                        }
243                }
244                
245                for (CompletableFuture<Map.Entry<Group, org.nasdanika.models.gitlab.Group>> gcf: groupCompletableFutures) {
246                        Map.Entry<Group,org.nasdanika.models.gitlab.Group> groupEntry = gcf.join();
247                        Group group = groupEntry.getKey();
248                        org.nasdanika.models.gitlab.Group modelGroup = groupEntry.getValue();
249                        if (!groupProvider.apply(group.getId()).complete(modelGroup)) {
250                                progressMonitor.worked(1, "Group completable future already completed for " + group.getId() + " " + group.getFullName(), group);
251                        }
252                        Long parentId = group.getParentId();
253                        if (parentId == null) {
254                                rootGroups.add(modelGroup);
255                        } else {
256                                groupProvider.apply(parentId).thenAccept(pg -> {
257                                        EList<org.nasdanika.models.gitlab.Group> subGroups = pg.getSubGroups();
258                                        synchronized (LoaderOld.this) { // Global synchronization - to avoid concurrency issues in notifications. Not needed here, just in case.
259                                                subGroups.add(modelGroup);
260                                        }
261                                });                                     
262                        }                       
263                }
264                
265                long incomplete = groupMap.values().stream().filter(cf -> !cf.isDone()).count();
266                if (incomplete > 0) {
267                        progressMonitor.worked(1, "There are incomplete set parent futures (orphan child groups): " + incomplete);
268                        for (org.nasdanika.models.gitlab.Group childGroup: childGroups) {
269                                rootGroups.add(childGroup);
270                        }
271                }
272                long exceptionally = groupMap.values().stream().filter(cf -> cf.isCompletedExceptionally()).count();
273                if (exceptionally > 0) {
274                        progressMonitor.worked(1, "There are exceptionally completed set parent futures: " + exceptionally);
275                        for (CompletableFuture<org.nasdanika.models.gitlab.Group> cf: groupMap.values()) {
276                                if (cf.isCompletedExceptionally()) {
277                                        cf.exceptionally(th -> {
278                                                progressMonitor.worked(1, "Exceptional completion: " + th, th);
279                                                return null;
280                                        });
281                                }
282                        }
283                }               
284                return rootGroups;
285        }
286
287        protected org.nasdanika.models.gitlab.Group createGroup(
288                        org.gitlab4j.api.models.Group group, 
289                        GroupApi groupApi,
290                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
291                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider, 
292                        ProgressMonitor progressMonitor) {
293                
294                return getFactory().createGroup();
295        }
296        
297        protected org.nasdanika.models.gitlab.Group loadGroup(
298                        org.gitlab4j.api.models.Group group, 
299                        GroupApi groupApi,
300                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
301                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider, 
302                        ProgressMonitor progressMonitor) {
303                
304                org.nasdanika.models.gitlab.Group modelGroup = createGroup(group, groupApi, groupProvider, projectProvider, progressMonitor); 
305                modelGroup.setAvatarUrl(group.getAvatarUrl());
306                modelGroup.setCreatedAt(group.getCreatedAt());
307                modelGroup.setDescription(group.getDescription());
308                modelGroup.setFullName(group.getFullName());
309                modelGroup.setFullPath(group.getFullPath());
310                modelGroup.setId(group.getId());
311                Statistics stats = group.getStatistics();
312                if (stats != null) {
313                        modelGroup.setJobArtifactsSize(stats.getJobArtifactsSize());
314                        modelGroup.setLfsObjectsSize(stats.getLfsObjectsSize());
315                        modelGroup.setRepositorySize(stats.getRepositorySize());
316                        modelGroup.setStorageSize(stats.getStorageSize());
317                }
318                modelGroup.setLfsEnabled(group.getLfsEnabled());
319                modelGroup.setName(group.getName());
320                modelGroup.setPath(group.getPath());
321                modelGroup.setVisibility(org.nasdanika.models.gitlab.Visibility.get(group.getVisibility().ordinal()));
322                modelGroup.setWebUrl(group.getWebUrl());
323                
324                EList<org.nasdanika.models.gitlab.Project> modelGroupProjects = modelGroup.getProjects();
325                
326                try {
327                        List<Project> groupProjects = groupApi.getProjects(group.getId());
328                        List<Member> groupMembers = groupApi.getMembers(group.getId());
329                        try (ProgressMonitor scaledGroupMonitor = progressMonitor.scale(1 + groupProjects.size() + groupMembers.size())) {
330                                scaledGroupMonitor.worked(Status.INFO, 1, "Retrieved " + groupProjects.size() + " projects and " + groupMembers.size() + " members");                                                   
331                                for (org.gitlab4j.api.models.Project project: groupProjects) {
332                                        try (ProgressMonitor projectMonitor = scaledGroupMonitor.split("Loading project " + project.getName() + " " + project.getId(), 1, project)) {
333                                                if (Objects.equals(project.getNamespace().getId(), group.getId())) {
334                                                        org.nasdanika.models.gitlab.Project modelProject = loadProject(
335                                                                        project, 
336                                                                        groupProvider, 
337                                                                        projectProvider,
338                                                                        projectMonitor);
339                                                        modelGroupProjects.add(modelProject);
340                                                }
341                                        }
342                                }
343                                EList<org.nasdanika.models.gitlab.Member> modelGroupMembers = modelGroup.getMembers();
344                                for (org.gitlab4j.api.models.Member member: groupMembers) {
345                                        try (ProgressMonitor memberMonitor = scaledGroupMonitor.split("Loading member " + member.getName() + " " + member.getId(), 1, member)) {                                
346                                                org.nasdanika.models.gitlab.Member modelMember = loadMember(member, memberMonitor);
347                                                modelGroupMembers.add(modelMember);
348                                        }
349                                }
350                        }
351        
352                } catch (GitLabApiException e) {
353                        progressMonitor.worked(Status.ERROR, 1, "Failed to load group", group, e);                      
354                }
355                return modelGroup;
356        }
357        
358        /**
359         * Creates a new instance of model project. Called by loadProject(). 
360         * This implementation calls getFactory().createProject(). Override to customize creation. 
361         * E.g. create a subclass of Project, load a project from a prototypes with some information pre-filled, ...
362         * @param project
363         * @param groupProvider
364         * @param projectProvider
365         * @param progressMonitor
366         * @return
367         */
368        protected org.nasdanika.models.gitlab.Project createProject(
369                        org.gitlab4j.api.models.Project project, 
370                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
371                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider,                 
372                        ProgressMonitor progressMonitor) {
373                return getFactory().createProject();
374        }       
375        
376        protected org.nasdanika.models.gitlab.Owner createOwner(Owner owner, ProgressMonitor progressMonitor) {
377                return getFactory().createOwner();
378        }
379        
380        protected org.nasdanika.models.gitlab.Branch createBranch(Branch branch, ProgressMonitor progressMonitor) {
381                return getFactory().createBranch();
382        }
383        
384        protected org.nasdanika.models.gitlab.Contributor createContributor(Contributor contributor, ProgressMonitor progressMonitor) {
385                return getFactory().createContributor();
386        }
387        
388        protected org.nasdanika.models.gitlab.Project loadProject(
389                        org.gitlab4j.api.models.Project project, 
390                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
391                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider,                 
392                        ProgressMonitor progressMonitor) {
393                                
394                org.nasdanika.models.gitlab.Project modelProject = createProject(project, groupProvider, projectProvider, progressMonitor);
395                
396                modelProject.setId(project.getId());
397                modelProject.setApprovalsBeforeMerge(project.getApprovalsBeforeMerge());
398                modelProject.setArchived(project.getArchived());
399                modelProject.setAvatarUrl(project.getAvatarUrl());
400                modelProject.setContainerRegistryEnabled(project.getContainerRegistryEnabled());
401                modelProject.setCreatedAt(project.getCreatedAt());
402                modelProject.setCreatorId(project.getCreatorId());
403                modelProject.setDefaultBranch(project.getDefaultBranch());
404                modelProject.setDescription(project.getDescription());          
405                modelProject.setForksCount(project.getForksCount());            
406
407                Project forkedFrom = project.getForkedFromProject();
408                if (forkedFrom != null) {
409                        projectProvider.apply(forkedFrom.getId()).thenAccept(modelProject::setForkedFrom);
410                }
411                
412                modelProject.setHttpUrlToRepo(project.getHttpUrlToRepo());              
413                modelProject.setIsPublic(project.getPublic());          
414                modelProject.setIssuesEnabled(project.getIssuesEnabled());              
415                modelProject.setJobsEnabled(project.getJobsEnabled());          
416                modelProject.setLastsActivityAt(project.getLastActivityAt());           
417                modelProject.setLfsEnabled(project.getLfsEnabled());            
418                modelProject.setMergeMethod(MergeMethod.get(project.getMergeMethod().ordinal()));               
419                modelProject.setMergeRequestsEnabled(project.getMergeRequestsEnabled());                
420                modelProject.setName(project.getName());                
421                modelProject.setNameWithNamespace(project.getNameWithNamespace());              
422                modelProject.setOnlyAllowMergeIfAllDiscussionsAreResolved(project.getOnlyAllowMergeIfAllDiscussionsAreResolved());              
423                modelProject.setOpenIssuesCount(project.getOpenIssuesCount());          
424                
425                Owner owner = project.getOwner();
426                if (owner != null) {
427                        org.nasdanika.models.gitlab.Owner modelOwner = createOwner(owner, progressMonitor);
428                        populateAbstractUser(owner, modelOwner);
429                        modelProject.setOwner(modelOwner);
430                }
431                
432                modelProject.setPath(project.getPath());
433                modelProject.setPathWithNamespace(project.getPathWithNamespace());
434                
435                Permissions permissions = project.getPermissions();
436                if (permissions != null) {
437                        ProjectAccess groupAccess = permissions.getGroupAccess();
438                        if (groupAccess != null) {
439                                org.nasdanika.models.gitlab.ProjectAccess modelGroupAccess = getFactory().createProjectAccess();
440                                modelGroupAccess.setAccessLevel(org.nasdanika.models.gitlab.AccessLevel.get(groupAccess.getAccessLevel().value));
441                                modelProject.setGroupAccess(modelGroupAccess);
442                        }
443                        ProjectAccess projectAccess = permissions.getProjectAccess();
444                        if (projectAccess != null) {
445                                org.nasdanika.models.gitlab.ProjectAccess modelProjectAccess = getFactory().createProjectAccess();
446                                modelProjectAccess.setAccessLevel(org.nasdanika.models.gitlab.AccessLevel.get(projectAccess.getAccessLevel().value));
447                                modelProject.setGroupAccess(modelProjectAccess);
448                        }
449                }
450                
451                modelProject.setPublicJobs(project.getPublicJobs());
452                modelProject.setRepositoryStorage(project.getRepositoryStorage());
453                modelProject.setRequestAccessEnabled(project.getRequestAccessEnabled());
454                modelProject.setRunnersToken(project.getRunnersToken());
455                modelProject.setSharedRunnersEnabled(project.getSharedRunnersEnabled());
456                
457                List<ProjectSharedGroup> sharedGroups = project.getSharedWithGroups();
458                if (sharedGroups != null) {
459                        for (ProjectSharedGroup sg: sharedGroups) {
460                                org.nasdanika.models.gitlab.ProjectSharedGroup psg = getFactory().createProjectSharedGroup();
461                                org.gitlab4j.api.models.AccessLevel accessLevel = sg.getGroupAccessLevel();
462                                if (accessLevel != null) {
463                                        psg.setAccessLevel(org.nasdanika.models.gitlab.AccessLevel.get(accessLevel.value));
464                                }
465                                psg.setFullPath(sg.getGroupFullPath());
466                                psg.setId(sg.getGroupId());
467                                psg.setName(sg.getGroupName());
468                                groupProvider.apply(sg.getGroupId()).thenAccept(psg::setGroup);
469                        }
470                }
471                                
472                modelProject.setSnippetsEnabled(project.getSnippetsEnabled());
473                modelProject.setSshUrlToRepo(project.getSshUrlToRepo());
474                modelProject.setStarCount(project.getStarCount());
475
476                List<String> tags = project.getTagList();
477                if (tags != null) {
478                        modelProject.getTags().addAll(tags);
479                }
480                
481                modelProject.setVisibilityLevel(project.getVisibilityLevel());
482                Visibility projectVisibility = project.getVisibility();
483                if (projectVisibility != null) {
484                        modelProject.setVisibility(org.nasdanika.models.gitlab.Visibility.get(projectVisibility.ordinal()));
485                }
486                modelProject.setWallEnabled(project.getWallEnabled());
487                modelProject.setWebUrl(project.getWebUrl());
488                modelProject.setWikiEnabled(project.getWikiEnabled());
489                modelProject.setPrintingMergeRequestLinkEnabled(project.getPrintingMergeRequestLinkEnabled());
490                modelProject.setResolveOutdatedDiffDiscussions(project.getResolveOutdatedDiffDiscussions());
491                
492                ProjectStatistics projectStatistics = project.getStatistics();
493                if (projectStatistics != null) {
494                        org.nasdanika.models.gitlab.ProjectStatistics modelProjectStatistics = getFactory().createProjectStatistics();
495                        modelProjectStatistics.setCommitCount(projectStatistics.getCommitCount());
496                        modelProjectStatistics.setJobArtifactsSize(projectStatistics.getJobArtifactsSize());
497                        modelProjectStatistics.setLfsObjectsSize(projectStatistics.getLfsObjectsSize());
498                        modelProjectStatistics.setPackagesSize(projectStatistics.getPackagesSize());
499                        modelProjectStatistics.setRepositorySize(projectStatistics.getRepositorySize());
500                        modelProjectStatistics.setStorageSize(projectStatistics.getStorageSize());
501                        modelProjectStatistics.setWikiSize(projectStatistics.getWikiSize());
502                        
503                        modelProject.setStatistics(modelProjectStatistics);
504                }
505                
506                modelProject.setInitializeWithReadme(null);
507                modelProject.setPackagesEnabled(null);
508                modelProject.setEmptyRepo(null);
509                modelProject.setLicenseUrl(null);
510                
511                ProjectLicense apiLicense = project.getLicense();
512                if (apiLicense != null) {
513                        modelProject.setLicense(loadProjectLicense(apiLicense, progressMonitor));
514                }
515
516                List<CustomAttribute> apiCustomAttributes = project.getCustomAttributes();
517                if (apiCustomAttributes != null) {
518                        EMap<String, String> modelCustomAttributes = modelProject.getCustomAttributes();
519                        apiCustomAttributes.forEach(ca -> modelCustomAttributes.put(ca.getKey(), ca.getValue()));
520                }
521                
522                modelProject.setBuildCoverageRegex(project.getBuildCoverageRegex());
523                
524                BuildGitStrategy buildGitStrategy = project.getBuildGitStrategy();
525                if (buildGitStrategy != null) {
526                        modelProject.setBuildGitStrategy(org.nasdanika.models.gitlab.BuildGitStrategy.get(buildGitStrategy.ordinal()));
527                }
528                
529                modelProject.setReadmeUrl(project.getReadmeUrl());
530                modelProject.setCanCreateMergeRequestIn(project.getCanCreateMergeRequestIn());
531                
532                org.gitlab4j.api.models.ImportStatus.Status importStatus = project.getImportStatus();
533                if (importStatus != null) {
534                        modelProject.setImportStatus(org.nasdanika.models.gitlab.Status.get(importStatus.ordinal()));
535                }
536                modelProject.setCiDefaultGitDepth(project.getCiDefaultGitDepth());
537                modelProject.setCiForwardDeploymentEnabled(project.getCiForwardDeploymentEnabled());
538                modelProject.setCiConfigPath(project.getCiConfigPath());
539                modelProject.setRemoveSourceBranchAfterMerge(project.getRemoveSourceBranchAfterMerge());
540                modelProject.setAutoDevopsEnabled(project.getAutoDevopsEnabled());
541                
542                AutoDevopsDeployStrategy autoDevopsDeployStrategy = project.getAutoDevopsDeployStrategy();
543                if (autoDevopsDeployStrategy != null) {
544                        modelProject.setAutoDevopsDeployStrategy(org.nasdanika.models.gitlab.AutoDevopsDeployStrategy.get(autoDevopsDeployStrategy.ordinal()));
545                }
546                
547                modelProject.setAutocloseReferencedIssues(project.getAutocloseReferencedIssues());
548                modelProject.setEmailsDisabled(project.getEmailsDisabled());
549                modelProject.setSuggestionCommitMessage(project.getSuggestionCommitMessage());
550                
551                SquashOption squashOption = project.getSquashOption();
552                if (squashOption != null) {
553                        modelProject.setSquashOption(org.nasdanika.models.gitlab.SquashOption.get(squashOption.ordinal()));
554                }               
555                
556                RepositoryApi repoApi = gitLabApi.getRepositoryApi();
557                try {
558                        List<Branch> branches = repoApi.getBranches(project.getId());
559                        if (branches != null) {
560                                EList<org.nasdanika.models.gitlab.Branch> modelBranches = modelProject.getBranches();
561                                for (Branch branch: branches) {
562                                        org.nasdanika.models.gitlab.Branch modelBranch = createBranch(branch, progressMonitor);
563                                        modelBranch.setCanPush(branch.getCanPush());
564                                        
565                                        Commit commit = branch.getCommit();
566                                        if (commit != null) {
567                                                modelBranch.setCommitDate(commit.getCommittedDate());
568                                        }
569                                        
570                                        modelBranch.setDevelopersCanMerge(branch.getDevelopersCanMerge());
571                                        modelBranch.setDevelopersCanPush(branch.getDevelopersCanPush());
572                                        modelBranch.setIsDefault(branch.getDefault());
573                                        modelBranch.setIsProtected(branch.getProtected());
574                                        modelBranch.setMerged(branch.getMerged());
575                                        modelBranch.setName(branch.getName());
576                                        modelBranch.setWebUrl(branch.getWebUrl());
577                                        modelBranches.add(modelBranch);
578                                        
579                                        if (isLoadPath(modelProject, modelBranch, ROOT_PATH)) {
580                                                modelBranch.getTreeItems().addAll(
581                                                                loadTree(
582                                                                                modelProject, 
583                                                                                modelBranch, 
584                                                                                ROOT_PATH, 
585                                                                                groupProvider, 
586                                                                                projectProvider,
587                                                                                progressMonitor));
588                                        }
589                                }                       
590                        }
591                } catch (GitLabApiException e) {
592                        progressMonitor.worked(Status.ERROR, 1, "Failed to load branches", project, e);
593                }
594                
595                try {
596                        List<Contributor> contributors = repoApi.getContributors(project.getId());
597                        if (contributors != null) {
598                                EList<org.nasdanika.models.gitlab.Contributor> modelContributors = modelProject.getContributors();
599                                for (Contributor contributor: contributors) {
600                                        org.nasdanika.models.gitlab.Contributor modelContributor = createContributor(contributor, progressMonitor);
601                                        populateAbstractUser(contributor, modelContributor);
602                                        modelContributor.setAdditions(contributor.getAdditions());
603                                        modelContributor.setCommits(contributor.getCommits());
604                                        modelContributor.setDeletions(contributor.getDeletions());
605                                        modelContributors.add(modelContributor);
606                                }                       
607                        }
608                } catch (GitLabApiException e) {
609                        progressMonitor.worked(Status.ERROR, 1, "Failed to load contributors", project, e);
610                }
611                
612                try {
613                        ProjectApi projectApi = gitLabApi.getProjectApi();
614                        List<Member> projectMembers = projectApi.getMembers(project.getId());                   
615                        EList<org.nasdanika.models.gitlab.Member> modelProjectMembers = modelProject.getMembers();
616                        for (org.gitlab4j.api.models.Member member: projectMembers) {
617                                org.nasdanika.models.gitlab.Member modelMember = loadMember(member, progressMonitor);
618                                modelProjectMembers.add(modelMember);
619                        }
620                } catch (GitLabApiException e) {
621                        progressMonitor.worked(Status.ERROR, 1, "Failed to load members", project, e);                  
622                }
623                                
624//              CommitsApi commitsApi = gitLabApi.getCommitsApi();
625//              for (Commit commit: commitsApi.getCommits(project.getId())) {
626//                      System.out.println("Commit: " + commit);
627//              }
628                        
629                return modelProject;
630        }       
631        
632        /**
633         * Loads branch tree
634         * @param project
635         * @param groupProvider
636         * @param projectProvider
637         * @param progressMonitor
638         * @return
639         * @throws GitLabApiException 
640         */
641        protected List<org.nasdanika.models.gitlab.TreeItem> loadTree(
642                        org.nasdanika.models.gitlab.Project modelProject,
643                        org.nasdanika.models.gitlab.Branch modelBranch,
644                        String path,
645                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
646                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider,                 
647                        ProgressMonitor progressMonitor) throws GitLabApiException {
648                
649                List<org.nasdanika.models.gitlab.TreeItem> ret = new ArrayList<>();
650                RepositoryApi repoApi = gitLabApi.getRepositoryApi();
651                for (TreeItem treeItem: repoApi.getTree(modelProject.getId(), path, modelBranch.getName())) {
652                        if (isLoadPath(modelProject, modelBranch, path)) {
653                                switch (treeItem.getType()) {
654                                case TREE:
655                                        org.nasdanika.models.gitlab.Tree subTree = createTree(modelProject, modelBranch, treeItem);
656                                        subTree.setId(treeItem.getId());
657                                        subTree.setName(treeItem.getName());
658                                        subTree.setPath(treeItem.getPath());
659                                        subTree.getTreeItems().addAll(
660                                                        loadTree(
661                                                                        modelProject, 
662                                                                        modelBranch, 
663                                                                        treeItem.getPath(), 
664                                                                        groupProvider, 
665                                                                        projectProvider, 
666                                                                        progressMonitor));
667                                        ret.add(subTree);
668                                        break;
669                                case BLOB:
670                                        org.nasdanika.models.gitlab.Blob blob = createBlob(
671                                                        modelProject, 
672                                                        modelBranch, 
673                                                        treeItem, 
674                                                        groupProvider, 
675                                                        projectProvider, 
676                                                        progressMonitor);
677                                        if (blob != null) {
678                                                blob.setId(treeItem.getId());
679                                                blob.setName(treeItem.getName());
680                                                blob.setPath(treeItem.getPath());
681                                                
682                                                ret.add(blob);
683                                        }
684                                        break;
685                                case COMMIT:
686                                        break;
687                                default:
688                                        break;
689                                
690                                }
691                        }
692                }
693                
694                return ret;
695        }
696        
697        /**
698         * Creates a model tree. Override to create specialized trees.
699         * @param modelProject
700         * @param modelBranch
701         * @param tree
702         * @return
703         */
704        protected org.nasdanika.models.gitlab.Tree createTree(
705                        org.nasdanika.models.gitlab.Project modelProject,
706                        org.nasdanika.models.gitlab.Branch modelBranch,
707                        TreeItem tree) {
708                
709                return getFactory().createTree();
710        }
711                
712        /**
713         * Returns true if a tree items at the specified path shall be loaded. This method returns false.
714         * Override to load items of interest.
715         * @param modelProject
716         * @param modelBranch
717         * @param path
718         * @return 
719         */
720        protected boolean isLoadPath(
721                        org.nasdanika.models.gitlab.Project modelProject,
722                        org.nasdanika.models.gitlab.Branch modelBranch,
723                        String path) {
724                return false;
725        }
726        
727        /**
728         * Creates and populates a model blob. This implementation returns RepositoryFile. Override to create specialized blobs, e.g. TextRepositoryFile.
729         * This method may return null or an instance of org.nasdanika.models.gitlab.Blob to avoid calling to repository file API. 
730         * @param modelProject
731         * @param modelBranch
732         * @param blob
733         * @return
734         * @throws GitLabApiException 
735         */
736        protected org.nasdanika.models.gitlab.Blob createBlob(
737                        org.nasdanika.models.gitlab.Project modelProject,
738                        org.nasdanika.models.gitlab.Branch modelBranch,
739                        TreeItem blob,
740                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
741                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider,                 
742                        ProgressMonitor progressMonitor) throws GitLabApiException {
743
744                RepositoryFileApi repoFileApi = gitLabApi.getRepositoryFileApi();
745                RepositoryFile repoFile = repoFileApi.getFile(modelProject.getId(), blob.getPath(), modelBranch.getName());
746                org.nasdanika.models.gitlab.RepositoryFile ret = createRepositoryFile(
747                                modelProject, 
748                                modelBranch, 
749                                blob, 
750                                repoFile, 
751                                groupProvider, 
752                                projectProvider, 
753                                progressMonitor);
754                
755                ret.setCommitId(ret.getCommitId());
756                ret.setLastCommitId(ret.getLastCommitId());
757                ret.setRef(ret.getRef());
758                ret.setSize(ret.getSize());
759                return ret;
760        }
761        
762        /**
763         * Creates and populates a model repository file. This implementation returns RepositoryFile. Override to create specialized blobs, e.g. TextRepositoryFile.
764         * @param modelProject
765         * @param modelBranch
766         * @param blob
767         * @param repositoryFile
768         * @return
769         */
770        protected org.nasdanika.models.gitlab.RepositoryFile createRepositoryFile(
771                        org.nasdanika.models.gitlab.Project modelProject,
772                        org.nasdanika.models.gitlab.Branch modelBranch,
773                        TreeItem blob,
774                        RepositoryFile repositoryFile,
775                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Group>> groupProvider,
776                        Function<Long, CompletableFuture<org.nasdanika.models.gitlab.Project>> projectProvider,                 
777                        ProgressMonitor progressMonitor) {
778
779                return getFactory().createRepositoryFile();
780        }
781        
782        /**
783         * 
784         * @param member
785         * @param userProvider Provides a user instance to reference by the member
786         * @param progressMonitor
787         * @return
788         */
789        protected org.nasdanika.models.gitlab.Member loadMember(org.gitlab4j.api.models.Member member, ProgressMonitor progressMonitor) {
790                org.nasdanika.models.gitlab.Member modelMember = getFactory().createMember();
791                populateAbstractUser(member, modelMember);
792                org.gitlab4j.api.models.AccessLevel accessLevel = member.getAccessLevel();
793                if (accessLevel != null) {
794                        modelMember.setAccessLevel(org.nasdanika.models.gitlab.AccessLevel.get(accessLevel.value));
795                }
796                modelMember.setExpiresAt(member.getExpiresAt());                
797                return modelMember;
798        }       
799        
800        @Override
801        public void close() {
802                if (gitLabApi != null) {
803                        gitLabApi.close();
804                }
805        }
806
807}