How to use resolution method in ATX

Best Python code snippet using ATX

resolutionCache.ts

Source:resolutionCache.ts Github

copy

Full Screen

1/*@internal*/2namespace ts {3 /** This is the cache of module/typedirectives resolution that can be retained across program */4 export interface ResolutionCache {5 startRecordingFilesWithChangedResolutions(): void;6 finishRecordingFilesWithChangedResolutions(): Path[] | undefined;78 resolveModuleNames(moduleNames: string[], containingFile: string, reusedNames: string[] | undefined, redirectedReference?: ResolvedProjectReference): (ResolvedModuleFull | undefined)[];9 getResolvedModuleWithFailedLookupLocationsFromCache(moduleName: string, containingFile: string): CachedResolvedModuleWithFailedLookupLocations | undefined;10 resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[];1112 invalidateResolutionOfFile(filePath: Path): void;13 removeResolutionsOfFile(filePath: Path): void;14 removeResolutionsFromProjectReferenceRedirects(filePath: Path): void;15 setFilesWithInvalidatedNonRelativeUnresolvedImports(filesWithUnresolvedImports: Map<ReadonlyArray<string>>): void;16 createHasInvalidatedResolution(forceAllFilesAsInvalidated?: boolean): HasInvalidatedResolution;1718 startCachingPerDirectoryResolution(): void;19 finishCachingPerDirectoryResolution(): void;2021 updateTypeRootsWatch(): void;22 closeTypeRootsWatch(): void;2324 clear(): void;25 }2627 interface ResolutionWithFailedLookupLocations {28 readonly failedLookupLocations: ReadonlyArray<string>;29 isInvalidated?: boolean;30 refCount?: number;31 }3233 interface ResolutionWithResolvedFileName {34 resolvedFileName: string | undefined;35 }3637 interface CachedResolvedModuleWithFailedLookupLocations extends ResolvedModuleWithFailedLookupLocations, ResolutionWithFailedLookupLocations {38 }3940 interface CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations extends ResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolutionWithFailedLookupLocations {41 }4243 export interface ResolutionCacheHost extends ModuleResolutionHost {44 toPath(fileName: string): Path;45 getCanonicalFileName: GetCanonicalFileName;46 getCompilationSettings(): CompilerOptions;47 watchDirectoryOfFailedLookupLocation(directory: string, cb: DirectoryWatcherCallback, flags: WatchDirectoryFlags): FileWatcher;48 onInvalidatedResolution(): void;49 watchTypeRootsDirectory(directory: string, cb: DirectoryWatcherCallback, flags: WatchDirectoryFlags): FileWatcher;50 onChangedAutomaticTypeDirectiveNames(): void;51 getCachedDirectoryStructureHost(): CachedDirectoryStructureHost | undefined;52 projectName?: string;53 getGlobalCache?(): string | undefined;54 writeLog(s: string): void;55 maxNumberOfFilesToIterateForInvalidation?: number;56 getCurrentProgram(): Program | undefined;57 }5859 interface DirectoryWatchesOfFailedLookup {60 /** watcher for the directory of failed lookup */61 watcher: FileWatcher;62 /** ref count keeping this directory watch alive */63 refCount: number;64 /** is the directory watched being non recursive */65 nonRecursive?: boolean;66 }6768 interface DirectoryOfFailedLookupWatch {69 dir: string;70 dirPath: Path;71 nonRecursive?: boolean;72 }7374 export function isPathInNodeModulesStartingWithDot(path: Path) {75 return stringContains(path, "/node_modules/.");76 }7778 export const maxNumberOfFilesToIterateForInvalidation = 256;7980 type GetResolutionWithResolvedFileName<T extends ResolutionWithFailedLookupLocations = ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName = ResolutionWithResolvedFileName> =81 (resolution: T) => R | undefined;8283 export function createResolutionCache(resolutionHost: ResolutionCacheHost, rootDirForResolution: string | undefined, logChangesWhenResolvingModule: boolean): ResolutionCache {84 let filesWithChangedSetOfUnresolvedImports: Path[] | undefined;85 let filesWithInvalidatedResolutions: Map<true> | undefined;86 let filesWithInvalidatedNonRelativeUnresolvedImports: ReadonlyMap<ReadonlyArray<string>> | undefined;87 let allFilesHaveInvalidatedResolution = false;88 const nonRelativeExternalModuleResolutions = createMultiMap<ResolutionWithFailedLookupLocations>();8990 const getCurrentDirectory = memoize(() => resolutionHost.getCurrentDirectory!()); // TODO: GH#1821791 const cachedDirectoryStructureHost = resolutionHost.getCachedDirectoryStructureHost();9293 // The resolvedModuleNames and resolvedTypeReferenceDirectives are the cache of resolutions per file.94 // The key in the map is source file's path.95 // The values are Map of resolutions with key being name lookedup.96 const resolvedModuleNames = createMap<Map<CachedResolvedModuleWithFailedLookupLocations>>();97 const perDirectoryResolvedModuleNames: CacheWithRedirects<Map<CachedResolvedModuleWithFailedLookupLocations>> = createCacheWithRedirects();98 const nonRelativeModuleNameCache: CacheWithRedirects<PerModuleNameCache> = createCacheWithRedirects();99 const moduleResolutionCache = createModuleResolutionCacheWithMaps(100 perDirectoryResolvedModuleNames,101 nonRelativeModuleNameCache,102 getCurrentDirectory(),103 resolutionHost.getCanonicalFileName104 );105106 const resolvedTypeReferenceDirectives = createMap<Map<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations>>();107 const perDirectoryResolvedTypeReferenceDirectives: CacheWithRedirects<Map<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations>> = createCacheWithRedirects();108109 /**110 * These are the extensions that failed lookup files will have by default,111 * any other extension of failed lookup will be store that path in custom failed lookup path112 * This helps in not having to comb through all resolutions when files are added/removed113 * Note that .d.ts file also has .d.ts extension hence will be part of default extensions114 */115 const failedLookupDefaultExtensions = [Extension.Ts, Extension.Tsx, Extension.Js, Extension.Jsx, Extension.Json];116 const customFailedLookupPaths = createMap<number>();117118 const directoryWatchesOfFailedLookups = createMap<DirectoryWatchesOfFailedLookup>();119 const rootDir = rootDirForResolution && removeTrailingDirectorySeparator(getNormalizedAbsolutePath(rootDirForResolution, getCurrentDirectory()));120 const rootPath = (rootDir && resolutionHost.toPath(rootDir)) as Path; // TODO: GH#18217121122 // TypeRoot watches for the types that get added as part of getAutomaticTypeDirectiveNames123 const typeRootsWatches = createMap<FileWatcher>();124125 return {126 startRecordingFilesWithChangedResolutions,127 finishRecordingFilesWithChangedResolutions,128 // perDirectoryResolvedModuleNames and perDirectoryResolvedTypeReferenceDirectives could be non empty if there was exception during program update129 // (between startCachingPerDirectoryResolution and finishCachingPerDirectoryResolution)130 startCachingPerDirectoryResolution: clearPerDirectoryResolutions,131 finishCachingPerDirectoryResolution,132 resolveModuleNames,133 getResolvedModuleWithFailedLookupLocationsFromCache,134 resolveTypeReferenceDirectives,135 removeResolutionsFromProjectReferenceRedirects,136 removeResolutionsOfFile,137 invalidateResolutionOfFile,138 setFilesWithInvalidatedNonRelativeUnresolvedImports,139 createHasInvalidatedResolution,140 updateTypeRootsWatch,141 closeTypeRootsWatch,142 clear143 };144145 function getResolvedModule(resolution: CachedResolvedModuleWithFailedLookupLocations) {146 return resolution.resolvedModule;147 }148149 function getResolvedTypeReferenceDirective(resolution: CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations) {150 return resolution.resolvedTypeReferenceDirective;151 }152153 function isInDirectoryPath(dir: Path | undefined, file: Path) {154 if (dir === undefined || file.length <= dir.length) {155 return false;156 }157 return startsWith(file, dir) && file[dir.length] === directorySeparator;158 }159160 function clear() {161 clearMap(directoryWatchesOfFailedLookups, closeFileWatcherOf);162 customFailedLookupPaths.clear();163 nonRelativeExternalModuleResolutions.clear();164 closeTypeRootsWatch();165 resolvedModuleNames.clear();166 resolvedTypeReferenceDirectives.clear();167 allFilesHaveInvalidatedResolution = false;168 // perDirectoryResolvedModuleNames and perDirectoryResolvedTypeReferenceDirectives could be non empty if there was exception during program update169 // (between startCachingPerDirectoryResolution and finishCachingPerDirectoryResolution)170 clearPerDirectoryResolutions();171 }172173 function startRecordingFilesWithChangedResolutions() {174 filesWithChangedSetOfUnresolvedImports = [];175 }176177 function finishRecordingFilesWithChangedResolutions() {178 const collected = filesWithChangedSetOfUnresolvedImports;179 filesWithChangedSetOfUnresolvedImports = undefined;180 return collected;181 }182183 function isFileWithInvalidatedNonRelativeUnresolvedImports(path: Path): boolean {184 if (!filesWithInvalidatedNonRelativeUnresolvedImports) {185 return false;186 }187188 // Invalidated if file has unresolved imports189 const value = filesWithInvalidatedNonRelativeUnresolvedImports.get(path);190 return !!value && !!value.length;191 }192193 function createHasInvalidatedResolution(forceAllFilesAsInvalidated?: boolean): HasInvalidatedResolution {194 if (allFilesHaveInvalidatedResolution || forceAllFilesAsInvalidated) {195 // Any file asked would have invalidated resolution196 filesWithInvalidatedResolutions = undefined;197 return returnTrue;198 }199 const collected = filesWithInvalidatedResolutions;200 filesWithInvalidatedResolutions = undefined;201 return path => (!!collected && collected.has(path)) ||202 isFileWithInvalidatedNonRelativeUnresolvedImports(path);203 }204205 function clearPerDirectoryResolutions() {206 perDirectoryResolvedModuleNames.clear();207 nonRelativeModuleNameCache.clear();208 perDirectoryResolvedTypeReferenceDirectives.clear();209 nonRelativeExternalModuleResolutions.forEach(watchFailedLookupLocationOfNonRelativeModuleResolutions);210 nonRelativeExternalModuleResolutions.clear();211 }212213 function finishCachingPerDirectoryResolution() {214 allFilesHaveInvalidatedResolution = false;215 filesWithInvalidatedNonRelativeUnresolvedImports = undefined;216 clearPerDirectoryResolutions();217 directoryWatchesOfFailedLookups.forEach((watcher, path) => {218 if (watcher.refCount === 0) {219 directoryWatchesOfFailedLookups.delete(path);220 watcher.watcher.close();221 }222 });223 }224225 function resolveModuleName(moduleName: string, containingFile: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference): CachedResolvedModuleWithFailedLookupLocations {226 const primaryResult = ts.resolveModuleName(moduleName, containingFile, compilerOptions, host, moduleResolutionCache, redirectedReference);227 // return result immediately only if global cache support is not enabled or if it is .ts, .tsx or .d.ts228 if (!resolutionHost.getGlobalCache) {229 return primaryResult;230 }231232 // otherwise try to load typings from @types233 const globalCache = resolutionHost.getGlobalCache();234 if (globalCache !== undefined && !isExternalModuleNameRelative(moduleName) && !(primaryResult.resolvedModule && extensionIsTS(primaryResult.resolvedModule.extension))) {235 // create different collection of failed lookup locations for second pass236 // if it will fail and we've already found something during the first pass - we don't want to pollute its results237 const { resolvedModule, failedLookupLocations } = loadModuleFromGlobalCache(moduleName, resolutionHost.projectName, compilerOptions, host, globalCache);238 if (resolvedModule) {239 return { resolvedModule, failedLookupLocations: addRange(primaryResult.failedLookupLocations as string[], failedLookupLocations) };240 }241 }242243 // Default return the result from the first pass244 return primaryResult;245 }246247 function resolveNamesWithLocalCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(248 names: ReadonlyArray<string>,249 containingFile: string,250 redirectedReference: ResolvedProjectReference | undefined,251 cache: Map<Map<T>>,252 perDirectoryCacheWithRedirects: CacheWithRedirects<Map<T>>,253 loader: (name: string, containingFile: string, options: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference) => T,254 getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,255 shouldRetryResolution: (t: T) => boolean,256 reusedNames: ReadonlyArray<string> | undefined,257 logChanges: boolean): (R | undefined)[] {258259 const path = resolutionHost.toPath(containingFile);260 const resolutionsInFile = cache.get(path) || cache.set(path, createMap()).get(path)!;261 const dirPath = getDirectoryPath(path);262 const perDirectoryCache = perDirectoryCacheWithRedirects.getOrCreateMapOfCacheRedirects(redirectedReference);263 let perDirectoryResolution = perDirectoryCache.get(dirPath);264 if (!perDirectoryResolution) {265 perDirectoryResolution = createMap();266 perDirectoryCache.set(dirPath, perDirectoryResolution);267 }268 const resolvedModules: (R | undefined)[] = [];269 const compilerOptions = resolutionHost.getCompilationSettings();270 const hasInvalidatedNonRelativeUnresolvedImport = logChanges && isFileWithInvalidatedNonRelativeUnresolvedImports(path);271272 // All the resolutions in this file are invalidated if this file wasnt resolved using same redirect273 const program = resolutionHost.getCurrentProgram();274 const oldRedirect = program && program.getResolvedProjectReferenceToRedirect(containingFile);275 const unmatchedRedirects = oldRedirect ?276 !redirectedReference || redirectedReference.sourceFile.path !== oldRedirect.sourceFile.path :277 !!redirectedReference;278279 const seenNamesInFile = createMap<true>();280 for (const name of names) {281 let resolution = resolutionsInFile.get(name);282 // Resolution is valid if it is present and not invalidated283 if (!seenNamesInFile.has(name) &&284 allFilesHaveInvalidatedResolution || unmatchedRedirects || !resolution || resolution.isInvalidated ||285 // If the name is unresolved import that was invalidated, recalculate286 (hasInvalidatedNonRelativeUnresolvedImport && !isExternalModuleNameRelative(name) && shouldRetryResolution(resolution))) {287 const existingResolution = resolution;288 const resolutionInDirectory = perDirectoryResolution.get(name);289 if (resolutionInDirectory) {290 resolution = resolutionInDirectory;291 }292 else {293 resolution = loader(name, containingFile, compilerOptions, resolutionHost, redirectedReference);294 perDirectoryResolution.set(name, resolution);295 }296 resolutionsInFile.set(name, resolution);297 watchFailedLookupLocationsOfExternalModuleResolutions(name, resolution);298 if (existingResolution) {299 stopWatchFailedLookupLocationOfResolution(existingResolution);300 }301302 if (logChanges && filesWithChangedSetOfUnresolvedImports && !resolutionIsEqualTo(existingResolution, resolution)) {303 filesWithChangedSetOfUnresolvedImports.push(path);304 // reset log changes to avoid recording the same file multiple times305 logChanges = false;306 }307 }308 Debug.assert(resolution !== undefined && !resolution.isInvalidated);309 seenNamesInFile.set(name, true);310 resolvedModules.push(getResolutionWithResolvedFileName(resolution));311 }312313 // Stop watching and remove the unused name314 resolutionsInFile.forEach((resolution, name) => {315 if (!seenNamesInFile.has(name) && !contains(reusedNames, name)) {316 stopWatchFailedLookupLocationOfResolution(resolution);317 resolutionsInFile.delete(name);318 }319 });320321 return resolvedModules;322323 function resolutionIsEqualTo(oldResolution: T | undefined, newResolution: T | undefined): boolean {324 if (oldResolution === newResolution) {325 return true;326 }327 if (!oldResolution || !newResolution) {328 return false;329 }330 const oldResult = getResolutionWithResolvedFileName(oldResolution);331 const newResult = getResolutionWithResolvedFileName(newResolution);332 if (oldResult === newResult) {333 return true;334 }335 if (!oldResult || !newResult) {336 return false;337 }338 return oldResult.resolvedFileName === newResult.resolvedFileName;339 }340 }341342 function resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[] {343 return resolveNamesWithLocalCache<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolvedTypeReferenceDirective>(344 typeDirectiveNames, containingFile, redirectedReference,345 resolvedTypeReferenceDirectives, perDirectoryResolvedTypeReferenceDirectives,346 resolveTypeReferenceDirective, getResolvedTypeReferenceDirective,347 /*shouldRetryResolution*/ resolution => resolution.resolvedTypeReferenceDirective === undefined,348 /*reusedNames*/ undefined, /*logChanges*/ false349 );350 }351352 function resolveModuleNames(moduleNames: string[], containingFile: string, reusedNames: string[] | undefined, redirectedReference?: ResolvedProjectReference): (ResolvedModuleFull | undefined)[] {353 return resolveNamesWithLocalCache<CachedResolvedModuleWithFailedLookupLocations, ResolvedModuleFull>(354 moduleNames, containingFile, redirectedReference,355 resolvedModuleNames, perDirectoryResolvedModuleNames,356 resolveModuleName, getResolvedModule,357 /*shouldRetryResolution*/ resolution => !resolution.resolvedModule || !resolutionExtensionIsTSOrJson(resolution.resolvedModule.extension),358 reusedNames, logChangesWhenResolvingModule359 );360 }361362 function getResolvedModuleWithFailedLookupLocationsFromCache(moduleName: string, containingFile: string): CachedResolvedModuleWithFailedLookupLocations | undefined {363 const cache = resolvedModuleNames.get(resolutionHost.toPath(containingFile));364 return cache && cache.get(moduleName);365 }366367 function isNodeModulesDirectory(dirPath: Path) {368 return endsWith(dirPath, "/node_modules");369 }370371 function isNodeModulesAtTypesDirectory(dirPath: Path) {372 return endsWith(dirPath, "/node_modules/@types");373 }374375 /**376 * Filter out paths like377 * "/", "/user", "/user/username", "/user/username/folderAtRoot",378 * "c:/", "c:/users", "c:/users/username", "c:/users/username/folderAtRoot", "c:/folderAtRoot"379 * @param dirPath380 */381 function canWatchDirectory(dirPath: Path) {382 const rootLength = getRootLength(dirPath);383 if (dirPath.length === rootLength) {384 // Ignore "/", "c:/"385 return false;386 }387388 const nextDirectorySeparator = dirPath.indexOf(directorySeparator, rootLength);389 if (nextDirectorySeparator === -1) {390 // ignore "/user", "c:/users" or "c:/folderAtRoot"391 return false;392 }393394 if (dirPath.charCodeAt(0) !== CharacterCodes.slash &&395 dirPath.substr(rootLength, nextDirectorySeparator).search(/users/i) === -1) {396 // Paths like c:/folderAtRoot/subFolder are allowed397 return true;398 }399400 for (let searchIndex = nextDirectorySeparator + 1, searchLevels = 2; searchLevels > 0; searchLevels--) {401 searchIndex = dirPath.indexOf(directorySeparator, searchIndex) + 1;402 if (searchIndex === 0) {403 // Folder isnt at expected minimun levels404 return false;405 }406 }407 return true;408 }409410 function getDirectoryToWatchFailedLookupLocation(failedLookupLocation: string, failedLookupLocationPath: Path): DirectoryOfFailedLookupWatch | undefined {411 if (isInDirectoryPath(rootPath, failedLookupLocationPath)) {412 // Ensure failed look up is normalized path413 failedLookupLocation = isRootedDiskPath(failedLookupLocation) ? normalizePath(failedLookupLocation) : getNormalizedAbsolutePath(failedLookupLocation, getCurrentDirectory());414 Debug.assert(failedLookupLocation.length === failedLookupLocationPath.length, `FailedLookup: ${failedLookupLocation} failedLookupLocationPath: ${failedLookupLocationPath}`); // tslint:disable-line415 const subDirectoryInRoot = failedLookupLocationPath.indexOf(directorySeparator, rootPath.length + 1);416 if (subDirectoryInRoot !== -1) {417 // Instead of watching root, watch directory in root to avoid watching excluded directories not needed for module resolution418 return { dir: failedLookupLocation.substr(0, subDirectoryInRoot), dirPath: failedLookupLocationPath.substr(0, subDirectoryInRoot) as Path };419 }420 else {421 // Always watch root directory non recursively422 return { dir: rootDir!, dirPath: rootPath, nonRecursive: false }; // TODO: GH#18217423 }424 }425426 return getDirectoryToWatchFromFailedLookupLocationDirectory(427 getDirectoryPath(getNormalizedAbsolutePath(failedLookupLocation, getCurrentDirectory())),428 getDirectoryPath(failedLookupLocationPath)429 );430 }431432 function getDirectoryToWatchFromFailedLookupLocationDirectory(dir: string, dirPath: Path): DirectoryOfFailedLookupWatch | undefined {433 // If directory path contains node module, get the most parent node_modules directory for watching434 while (pathContainsNodeModules(dirPath)) {435 dir = getDirectoryPath(dir);436 dirPath = getDirectoryPath(dirPath);437 }438439 // If the directory is node_modules use it to watch, always watch it recursively440 if (isNodeModulesDirectory(dirPath)) {441 return canWatchDirectory(getDirectoryPath(dirPath)) ? { dir, dirPath } : undefined;442 }443444 let nonRecursive = true;445 // Use some ancestor of the root directory446 let subDirectoryPath: Path | undefined, subDirectory: string | undefined;447 if (rootPath !== undefined) {448 while (!isInDirectoryPath(dirPath, rootPath)) {449 const parentPath = getDirectoryPath(dirPath);450 if (parentPath === dirPath) {451 break;452 }453 nonRecursive = false;454 subDirectoryPath = dirPath;455 subDirectory = dir;456 dirPath = parentPath;457 dir = getDirectoryPath(dir);458 }459 }460461 return canWatchDirectory(dirPath) ? { dir: subDirectory || dir, dirPath: subDirectoryPath || dirPath, nonRecursive } : undefined;462 }463464 function isPathWithDefaultFailedLookupExtension(path: Path) {465 return fileExtensionIsOneOf(path, failedLookupDefaultExtensions);466 }467468 function watchFailedLookupLocationsOfExternalModuleResolutions(name: string, resolution: ResolutionWithFailedLookupLocations) {469 // No need to set the resolution refCount470 if (resolution.failedLookupLocations && resolution.failedLookupLocations.length) {471 if (resolution.refCount) {472 resolution.refCount++;473 }474 else {475 resolution.refCount = 1;476 if (isExternalModuleNameRelative(name)) {477 watchFailedLookupLocationOfResolution(resolution);478 }479 else {480 nonRelativeExternalModuleResolutions.add(name, resolution);481 }482 }483 }484 }485486 function watchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {487 Debug.assert(!!resolution.refCount);488489 const { failedLookupLocations } = resolution;490 let setAtRoot = false;491 for (const failedLookupLocation of failedLookupLocations) {492 const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);493 const toWatch = getDirectoryToWatchFailedLookupLocation(failedLookupLocation, failedLookupLocationPath);494 if (toWatch) {495 const { dir, dirPath, nonRecursive } = toWatch;496 // If the failed lookup location path is not one of the supported extensions,497 // store it in the custom path498 if (!isPathWithDefaultFailedLookupExtension(failedLookupLocationPath)) {499 const refCount = customFailedLookupPaths.get(failedLookupLocationPath) || 0;500 customFailedLookupPaths.set(failedLookupLocationPath, refCount + 1);501 }502 if (dirPath === rootPath) {503 Debug.assert(!nonRecursive);504 setAtRoot = true;505 }506 else {507 setDirectoryWatcher(dir, dirPath, nonRecursive);508 }509 }510 }511512 if (setAtRoot) {513 // This is always non recursive514 setDirectoryWatcher(rootDir!, rootPath, /*nonRecursive*/ true); // TODO: GH#18217515 }516 }517518 function setRefCountToUndefined(resolution: ResolutionWithFailedLookupLocations) {519 resolution.refCount = undefined;520 }521522 function watchFailedLookupLocationOfNonRelativeModuleResolutions(resolutions: ResolutionWithFailedLookupLocations[], name: string) {523 const program = resolutionHost.getCurrentProgram();524 const updateResolution = program && program.getTypeChecker().tryFindAmbientModuleWithoutAugmentations(name) ?525 setRefCountToUndefined : watchFailedLookupLocationOfResolution;526 resolutions.forEach(updateResolution);527 }528529 function setDirectoryWatcher(dir: string, dirPath: Path, nonRecursive?: boolean) {530 const dirWatcher = directoryWatchesOfFailedLookups.get(dirPath);531 if (dirWatcher) {532 Debug.assert(!!nonRecursive === !!dirWatcher.nonRecursive);533 dirWatcher.refCount++;534 }535 else {536 directoryWatchesOfFailedLookups.set(dirPath, { watcher: createDirectoryWatcher(dir, dirPath, nonRecursive), refCount: 1, nonRecursive });537 }538 }539540 function stopWatchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {541 if (!resolution.refCount) {542 return;543 }544545 resolution.refCount--;546 if (resolution.refCount) {547 return;548 }549550 const { failedLookupLocations } = resolution;551 let removeAtRoot = false;552 for (const failedLookupLocation of failedLookupLocations) {553 const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);554 const toWatch = getDirectoryToWatchFailedLookupLocation(failedLookupLocation, failedLookupLocationPath);555 if (toWatch) {556 const { dirPath } = toWatch;557 const refCount = customFailedLookupPaths.get(failedLookupLocationPath);558 if (refCount) {559 if (refCount === 1) {560 customFailedLookupPaths.delete(failedLookupLocationPath);561 }562 else {563 Debug.assert(refCount > 1);564 customFailedLookupPaths.set(failedLookupLocationPath, refCount - 1);565 }566 }567568 if (dirPath === rootPath) {569 removeAtRoot = true;570 }571 else {572 removeDirectoryWatcher(dirPath);573 }574 }575 }576 if (removeAtRoot) {577 removeDirectoryWatcher(rootPath);578 }579 }580581 function removeDirectoryWatcher(dirPath: string) {582 const dirWatcher = directoryWatchesOfFailedLookups.get(dirPath)!;583 // Do not close the watcher yet since it might be needed by other failed lookup locations.584 dirWatcher.refCount--;585 }586587 function createDirectoryWatcher(directory: string, dirPath: Path, nonRecursive: boolean | undefined) {588 return resolutionHost.watchDirectoryOfFailedLookupLocation(directory, fileOrDirectory => {589 const fileOrDirectoryPath = resolutionHost.toPath(fileOrDirectory);590 if (cachedDirectoryStructureHost) {591 // Since the file existence changed, update the sourceFiles cache592 cachedDirectoryStructureHost.addOrDeleteFileOrDirectory(fileOrDirectory, fileOrDirectoryPath);593 }594595 if (!allFilesHaveInvalidatedResolution && invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {596 resolutionHost.onInvalidatedResolution();597 }598 }, nonRecursive ? WatchDirectoryFlags.None : WatchDirectoryFlags.Recursive);599 }600601 function removeResolutionsOfFileFromCache(cache: Map<Map<ResolutionWithFailedLookupLocations>>, filePath: Path) {602 // Deleted file, stop watching failed lookups for all the resolutions in the file603 const resolutions = cache.get(filePath);604 if (resolutions) {605 resolutions.forEach(stopWatchFailedLookupLocationOfResolution);606 cache.delete(filePath);607 }608 }609610 function removeResolutionsFromProjectReferenceRedirects(filePath: Path) {611 if (!fileExtensionIs(filePath, Extension.Json)) { return; }612613 const program = resolutionHost.getCurrentProgram();614 if (!program) { return; }615616 // If this file is input file for the referenced project, get it617 const resolvedProjectReference = program.getResolvedProjectReferenceByPath(filePath);618 if (!resolvedProjectReference) { return; }619620 // filePath is for the projectReference and the containing file is from this project reference, invalidate the resolution621 resolvedProjectReference.commandLine.fileNames.forEach(f => removeResolutionsOfFile(resolutionHost.toPath(f)));622 }623624 function removeResolutionsOfFile(filePath: Path) {625 removeResolutionsOfFileFromCache(resolvedModuleNames, filePath);626 removeResolutionsOfFileFromCache(resolvedTypeReferenceDirectives, filePath);627 }628629 function invalidateResolutionCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(630 cache: Map<Map<T>>,631 isInvalidatedResolution: (resolution: T, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>) => boolean,632 getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>633 ) {634 const seen = createMap<Map<true>>();635 cache.forEach((resolutions, containingFilePath) => {636 const dirPath = getDirectoryPath(containingFilePath);637 let seenInDir = seen.get(dirPath);638 if (!seenInDir) {639 seenInDir = createMap<true>();640 seen.set(dirPath, seenInDir);641 }642 resolutions.forEach((resolution, name) => {643 if (seenInDir!.has(name)) {644 return;645 }646 seenInDir!.set(name, true);647 if (!resolution.isInvalidated && isInvalidatedResolution(resolution, getResolutionWithResolvedFileName)) {648 // Mark the file as needing re-evaluation of module resolution instead of using it blindly.649 resolution.isInvalidated = true;650 (filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);651 }652 });653 });654 }655656 function hasReachedResolutionIterationLimit() {657 const maxSize = resolutionHost.maxNumberOfFilesToIterateForInvalidation || maxNumberOfFilesToIterateForInvalidation;658 return resolvedModuleNames.size > maxSize || resolvedTypeReferenceDirectives.size > maxSize;659 }660661 function invalidateResolutions(662 isInvalidatedResolution: (resolution: ResolutionWithFailedLookupLocations, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName) => boolean,663 ) {664 // If more than maxNumberOfFilesToIterateForInvalidation present,665 // just invalidated all files and recalculate the resolutions for files instead666 if (hasReachedResolutionIterationLimit()) {667 allFilesHaveInvalidatedResolution = true;668 return;669 }670 invalidateResolutionCache(resolvedModuleNames, isInvalidatedResolution, getResolvedModule);671 invalidateResolutionCache(resolvedTypeReferenceDirectives, isInvalidatedResolution, getResolvedTypeReferenceDirective);672 }673674 function invalidateResolutionOfFile(filePath: Path) {675 removeResolutionsOfFile(filePath);676 invalidateResolutions(677 // Resolution is invalidated if the resulting file name is same as the deleted file path678 (resolution, getResolutionWithResolvedFileName) => {679 const result = getResolutionWithResolvedFileName(resolution);680 return !!result && resolutionHost.toPath(result.resolvedFileName!) === filePath; // TODO: GH#18217681 }682 );683 }684685 function setFilesWithInvalidatedNonRelativeUnresolvedImports(filesMap: ReadonlyMap<ReadonlyArray<string>>) {686 Debug.assert(filesWithInvalidatedNonRelativeUnresolvedImports === filesMap || filesWithInvalidatedNonRelativeUnresolvedImports === undefined);687 filesWithInvalidatedNonRelativeUnresolvedImports = filesMap;688 }689690 function invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath: Path, isCreatingWatchedDirectory: boolean) {691 let isChangedFailedLookupLocation: (location: string) => boolean;692 if (isCreatingWatchedDirectory) {693 // Watching directory is created694 // Invalidate any resolution has failed lookup in this directory695 isChangedFailedLookupLocation = location => isInDirectoryPath(fileOrDirectoryPath, resolutionHost.toPath(location));696 }697 else {698 // If something to do with folder/file starting with "." in node_modules folder, skip it699 if (isPathInNodeModulesStartingWithDot(fileOrDirectoryPath)) return false;700701 // Some file or directory in the watching directory is created702 // Return early if it does not have any of the watching extension or not the custom failed lookup path703 const dirOfFileOrDirectory = getDirectoryPath(fileOrDirectoryPath);704 if (isNodeModulesAtTypesDirectory(fileOrDirectoryPath) || isNodeModulesDirectory(fileOrDirectoryPath) ||705 isNodeModulesAtTypesDirectory(dirOfFileOrDirectory) || isNodeModulesDirectory(dirOfFileOrDirectory)) {706 // Invalidate any resolution from this directory707 isChangedFailedLookupLocation = location => {708 const locationPath = resolutionHost.toPath(location);709 return locationPath === fileOrDirectoryPath || startsWith(resolutionHost.toPath(location), fileOrDirectoryPath);710 };711 }712 else {713 if (!isPathWithDefaultFailedLookupExtension(fileOrDirectoryPath) && !customFailedLookupPaths.has(fileOrDirectoryPath)) {714 return false;715 }716 // Ignore emits from the program717 if (isEmittedFileOfProgram(resolutionHost.getCurrentProgram(), fileOrDirectoryPath)) {718 return false;719 }720 // Resolution need to be invalidated if failed lookup location is same as the file or directory getting created721 isChangedFailedLookupLocation = location => resolutionHost.toPath(location) === fileOrDirectoryPath;722 }723 }724 const hasChangedFailedLookupLocation = (resolution: ResolutionWithFailedLookupLocations) => some(resolution.failedLookupLocations, isChangedFailedLookupLocation);725 const invalidatedFilesCount = filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size;726 invalidateResolutions(727 // Resolution is invalidated if the resulting file name is same as the deleted file path728 hasChangedFailedLookupLocation729 );730 return allFilesHaveInvalidatedResolution || filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size !== invalidatedFilesCount;731 }732733 function closeTypeRootsWatch() {734 clearMap(typeRootsWatches, closeFileWatcher);735 }736737 function getDirectoryToWatchFailedLookupLocationFromTypeRoot(typeRoot: string, typeRootPath: Path): Path | undefined {738 if (allFilesHaveInvalidatedResolution) {739 return undefined;740 }741742 if (isInDirectoryPath(rootPath, typeRootPath)) {743 return rootPath;744 }745 const toWatch = getDirectoryToWatchFromFailedLookupLocationDirectory(typeRoot, typeRootPath);746 return toWatch && directoryWatchesOfFailedLookups.has(toWatch.dirPath) ? toWatch.dirPath : undefined;747 }748749 function createTypeRootsWatch(typeRootPath: Path, typeRoot: string): FileWatcher {750 // Create new watch and recursive info751 return resolutionHost.watchTypeRootsDirectory(typeRoot, fileOrDirectory => {752 const fileOrDirectoryPath = resolutionHost.toPath(fileOrDirectory);753 if (cachedDirectoryStructureHost) {754 // Since the file existence changed, update the sourceFiles cache755 cachedDirectoryStructureHost.addOrDeleteFileOrDirectory(fileOrDirectory, fileOrDirectoryPath);756 }757758 // For now just recompile759 // We could potentially store more data here about whether it was/would be really be used or not760 // and with that determine to trigger compilation but for now this is enough761 resolutionHost.onChangedAutomaticTypeDirectiveNames();762763 // Since directory watchers invoked are flaky, the failed lookup location events might not be triggered764 // So handle to failed lookup locations here as well to ensure we are invalidating resolutions765 const dirPath = getDirectoryToWatchFailedLookupLocationFromTypeRoot(typeRoot, typeRootPath);766 if (dirPath && invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {767 resolutionHost.onInvalidatedResolution();768 }769 }, WatchDirectoryFlags.Recursive);770 }771772 /**773 * Watches the types that would get added as part of getAutomaticTypeDirectiveNames774 * To be called when compiler options change775 */776 function updateTypeRootsWatch() {777 const options = resolutionHost.getCompilationSettings();778 if (options.types) {779 // No need to do any watch since resolution cache is going to handle the failed lookups780 // for the types added by this781 closeTypeRootsWatch();782 return;783 }784785 // we need to assume the directories exist to ensure that we can get all the type root directories that get included786 // But filter directories that are at root level to say directory doesnt exist, so that we arent watching them787 const typeRoots = getEffectiveTypeRoots(options, { directoryExists: directoryExistsForTypeRootWatch, getCurrentDirectory });788 if (typeRoots) {789 mutateMap(790 typeRootsWatches,791 arrayToMap(typeRoots, tr => resolutionHost.toPath(tr)),792 {793 createNewValue: createTypeRootsWatch,794 onDeleteValue: closeFileWatcher795 }796 );797 }798 else {799 closeTypeRootsWatch();800 }801 }802803 /**804 * Use this function to return if directory exists to get type roots to watch805 * If we return directory exists then only the paths will be added to type roots806 * Hence return true for all directories except root directories which are filtered from watching807 */808 function directoryExistsForTypeRootWatch(nodeTypesDirectory: string) {809 const dir = getDirectoryPath(getDirectoryPath(nodeTypesDirectory));810 const dirPath = resolutionHost.toPath(dir);811 return dirPath === rootPath || canWatchDirectory(dirPath);812 }813 } ...

Full Screen

Full Screen

Resampling.py

Source:Resampling.py Github

copy

Full Screen

1# -*- coding: utf-8 -*-2"""3The *Resampling* module provides methods to resample and reorient volumetric 4and point data. 5Resampling the data is usually necessary as the first step to match the 6resolution and orientation of the reference object. 7Main routines for resampling are: :func:`~ClearMap.Alignment.Resampling.resampleData` 8and :func:`~ClearMap.Alignment.Resampling.resamplePoints`.9Image Representation and Size10-----------------------------11The module assumes that images in arrays are arranged as 12 * [x,y] or 13 * [x,y,z] 14where x,y,z correspond to the x,y,z coordinates as displayed in e.g. ImageJ. 15For example an image of size (512,512) stored in an array ``img`` will have:16 >>> img.shape17 (512,512)18Points are assumed to be given as x,y,z coordinates19Parameters such as *resolution* or *dataSize* are assumed to be given in (x,y)20or (x,y,z) format, e.g.21 >>> dataSize = (512,512)22Orientation23-----------24The *orientation* parameter is a tuple of d numbers from 1 to d that specifies 25the permutation of the axes, a minus sign infront of a numbeer indicates26inversion of that axes. For exmaple 27 >>> orientation=(2,-1) 28indicates that x and y should be exchanged and the new y axes should be reversed.29Generally a re-orientation is composed of first a permutation of the axes and then30inverting the indicated axes.31A *permutation* is an orientation without signs and with numbers from 0 to d-1.32Examples:33 >>> import os34 >>> import ClearMap.IO as io35 >>> from ClearMap.Settings import ClearMapPath 36 >>> from ClearMap.Alignment.Resampling import resampleData37 >>> filename = os.path.join(ClearMapPath,'Test/Data/OME/16-17-27_0_8X-s3-20HF_UltraII_C00_xyz-Table Z\d{4}.ome.tif');38 >>> print io.dataSize(filename)39 (2160, 2560, 21)40 >>> data = resampleData(filename, sink = None, resolutionSource = (1,1,1), orientation = (1,2,3), resolutionSink = (10,10,2));41 >>> print data.shape42 (216, 256, 10)43"""44#:copyright: Copyright 2015 by Christoph Kirst, The Rockefeller University, New York City45#:license: GNU, see LICENSE.txt for details.46import sys, imp47import os48import math49import numpy50import multiprocessing 51import tempfile52import shutil53import cv254#import matplotlib.pyplot as plt55import ClearMap.IO.IO as io56import ClearMap.IO.FileList as fl57from ClearMap.Utils.ProcessWriter import ProcessWriter;58def fixOrientation(orientation):59 """Convert orientation to standard format number sequence60 61 Arguments:62 orientation (tuple or str): orientation specification63 64 Returns:65 tuple: orientation sequence66 67 See Also:68 `Orientation`_69 """70 71 if orientation is None:72 return None;73 74 #fix named representations75 if orientation == 'Left':76 orientation = (1,2,3);77 if orientation == 'Right':78 orientation = (-1,2,3); 79 80 return orientation;81def inverseOrientation(orientation):82 """Returns the inverse permuation of the permutation orientation taking axis inversions into account.83 84 Arguments:85 orientation (tuple or str): orientation specification86 87 Returns:88 tuple: orientation sequence89 90 See Also:91 `Orientation`_92 """93 94 if orientation is None:95 return None;96 97 n = len(orientation);98 iper = list(orientation);99 100 #permutation is defined as permuting the axes and then axis inversion101 for i in range(n):102 if orientation[i] < 0:103 iper[int(abs(orientation[i])-1)] = -(i + 1);104 else:105 iper[int(abs(orientation[i])-1)] = (i + 1);106 107 return tuple(iper)108def orientationToPermuation(orientation):109 """Extracts the permuation from an orientation.110 111 Arguments:112 orientation (tuple or str): orientation specification113 114 Returns:115 tuple: premutation sequence116 117 See Also:118 `Orientation`_119 """120 orientation = fixOrientation(orientation);121 if orientation is None:122 return (0,1,2);123 else:124 return tuple(int(abs(i))-1 for i in orientation);125def orientResolution(resolution, orientation):126 """Permutes a resolution tuple according to the given orientation.127 128 Arguments:129 resolution (tuple): resolution specification130 orientation (tuple or str): orientation specification131 132 Returns:133 tuple: oriented resolution sequence134 135 See Also:136 `Orientation`_137 """138 if resolution is None:139 return None;140 141 per = orientationToPermuation(orientation);142 #print orientation, per, resolution143 return tuple(resolution[i] for i in per);144 145def orientResolutionInverse(resolution, orientation):146 """Permutes a resolution tuple according to the inverse of a given orientation.147 148 Arguments:149 resolution (tuple): resolution specification150 orientation (tuple or str): orientation specification151 152 Returns:153 tuple: oriented resolution sequence154 155 See Also:156 `Orientation`_157 """158 159 if resolution is None:160 return None;161 162 per = orientationToPermuation(inverseOrientation(orientation));163 return tuple(resolution[i] for i in per);164 165def orientDataSize(dataSize, orientation):166 """Permutes a data size tuple according to the given orientation.167 168 Arguments:169 dataSize (tuple): resolution specification170 orientation (tuple or str): orientation specification171 172 Returns:173 tuple: oriented dataSize sequence174 175 See Also:176 `Orientation`_177 """178 179 return orientResolution(dataSize, orientation);180 181def orientDataSizeInverse(dataSize, orientation):182 """Permutes a dataSize tuple according to the inverse of a given orientation.183 184 Arguments:185 dataSize (tuple): dataSize specification186 orientation (tuple or str): orientation specification187 188 Returns:189 tuple: oriented dataSize sequence190 191 See Also:192 `Orientation`_193 """194 195 return orientResolutionInverse(dataSize, orientation); 196 197 198def resampleDataSize(dataSizeSource, dataSizeSink = None, resolutionSource = None, resolutionSink = None, orientation = None):199 """Calculate scaling factors and data sizes for resampling.200 201 Arguments:202 dataSizeSource (tuple): data size of the original image203 dataSizeSink (tuple or None): data size of the resmapled image204 resolutionSource (tuple or None): resolution of the source image205 resolutionSink (tuple or None): resolution of the sink image206 orientation (tuple or str): re-orientation specification207 208 Returns:209 tuple: data size of the source210 tuple: data size of the sink211 tuple: resolution of source212 tuple: resolution of sink213 214 See Also:215 `Orientation`_216 """217 orientation = fixOrientation(orientation); 218 219 #determine data sizes if not specified220 if dataSizeSink is None:221 if resolutionSource is None or resolutionSink is None:222 raise RuntimeError('resampleDataSize: data size and resolutions not defined!');223 224 #orient resolution of source to resolution of sink to get sink data size225 resolutionSourceO = orientResolution(resolutionSource, orientation);226 dataSizeSourceO = orientDataSize(dataSizeSource, orientation);227 228 #calculate scaling factor229 dataSizeSink = tuple([int(math.ceil(dataSizeSourceO[i] * resolutionSourceO[i]/resolutionSink[i])) for i in range(len(dataSizeSource))]); 230 231 #print dataSizeSink, "ds sink"232 233 if dataSizeSource is None:234 if resolutionSource is None or resolutionSink is None:235 raise RuntimeError('resampleDataSize: data size and resolutions not defined!');236 237 #orient resolution of source to resolution of sink to get sink data size238 resolutionSourceO = orientResolution(resolutionSource, orientation);239 240 #calculate source data size241 dataSizeSource = tuple([int(math.ceil(dataSizeSink[i] * resolutionSink[i]/resolutionSourceO[i])) for i in range(len(dataSizeSink))]); 242 dataSizeSource = orientDataSizeInverse(dataSizeSource);243 244 #print dataSizeSource, "ds source"245 246 #calculate effecive resolutions247 if resolutionSource is None:248 if resolutionSink is None:249 resolutionSource = (1,1,1);250 else:251 dataSizeSourceO = orientDataSize(dataSizeSource, orientation);252 resolutionSource = tuple(float(dataSizeSink[i]) / dataSizeSourceO[i] * resolutionSink[i] for i in range(len(dataSizeSource)));253 resolutionSource = orientResolutionInverse(resolutionSource, orientation);254 255 #print resolutionSource, "res source sink"256 257 dataSizeSourceO = orientDataSize(dataSizeSource, orientation);258 259 260 resolutionSourceO = orientResolution(resolutionSource, orientation);261 resolutionSink = tuple(float(dataSizeSourceO[i]) / float(dataSizeSink[i]) * resolutionSourceO[i] for i in range(len(dataSizeSource)));262 263 #print dataSizeSource, dataSizeSink, resolutionSource, resolutionSink 264 265 return dataSizeSource, dataSizeSink, resolutionSource, resolutionSink 266def fixInterpolation(interpolation):267 """Converts interpolation given as string to cv2 interpolation object268 269 Arguments:270 interpolation (str or object): interpolation string or cv2 object271 272 Returns:273 object: cv2 interpolation type274 """275 276 if interpolation == 'nn' or interpolation is None or interpolation == cv2.INTER_NEAREST:277 interpolation = cv2.INTER_NEAREST;278 else:279 interpolation = cv2.INTER_LINEAR;280 281 return interpolation;282 283def resampleXY(source, dataSizeSink, sink = None, interpolation = 'linear', out = sys.stdout, verbose = True):284 """Resample a 2d image slice285 286 This routine is used for resampling a large stack in parallel in xy or xz direction.287 288 Arguments:289 source (str or array): 2d image source290 dataSizeSink (tuple): size of the resmapled image291 sink (str or None): location for the resmapled image292 interpolation (str): interpolation method to use: 'linear' or None (nearest pixel)293 out (stdout): where to write progress information294 vebose (bool): write progress info if true295 296 Returns:297 array or str: resampled data or file name298 """ 299 300 #out.write("Input: %s Output: " % (inputFile, soutputFile))301 data = io.readData(source);302 dataSize = data.shape;303 304 #print dataSize, dataSizeSink 305 306 if data.ndim != 2:307 raise RuntimeError('resampleXY: expects 2d image source, found %dd' % data.ndim)308 #print sagittalImageSize;309 310 #dataSizeSink = tuple([int(math.ceil(dataSize[i] * resolutionSource[i]/resolutionSink[i])) for i in range(2)]);311 if verbose:312 out.write(("resampleData: Imagesize: %d, %d " % (dataSize[0], dataSize[1])) + ("Resampled Imagesize: %d, %d" % (dataSizeSink[0], dataSizeSink[1])))313 #out.write(("resampleData: Imagesize: %d, %d " % dataSize) + ("Resampled Imagesize: %d, %d" % (outputSize[1], outputSize[0])))314 315 # note: cv2.resize reverses x-Y axes316 interpolation = fixInterpolation(interpolation)317 sinkData = cv2.resize(data, (dataSizeSink[1], dataSizeSink[0]), interpolation = interpolation);318 #sinkData = cv2.resize(data, outputSize);319 #sinkData = scipy.misc.imresize(sagittalImage, outputImageSize, interp = 'bilinear'); #normalizes images -> not usefull for stacks !320 321 #out.write("resampleData: resized Image size: %d, %d " % sinkData.shape)322 323 return io.writeData(sink, sinkData);324def _resampleXYParallel(arg):325 """Resampling helper function to use for parallel resampling of image slices"""326 327 fileSource = arg[0];328 fileSink = arg[1];329 dataSizeSink = arg[2];330 interpolation = arg[3];331 ii = arg[4];332 nn = arg[5];333 verbose = arg[6];334 335 pw = ProcessWriter(ii);336 if verbose:337 pw.write("resampleData: resampling in XY: image %d / %d" % (ii, nn))338 339 data = numpy.squeeze(io.readData(fileSource, z = ii));340 resampleXY(data, sink = fileSink, dataSizeSink = dataSizeSink, interpolation = interpolation, out = pw, verbose = verbose);341def resampleData(source, sink = None, orientation = None, dataSizeSink = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), 342 processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):343 """Resample data of source in resolution and orientation344 345 Arguments:346 source (str or array): image to be resampled347 sink (str or None): destination of resampled image348 orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)349 dataSizeSink (tuple or None): target size of the resampled image350 resolutionSource (tuple): resolution of the source image (in length per pixel)351 resolutionSink (tuple): resolution of the resampled image (in length per pixel)352 processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created353 processes (int): number of processes to use for parallel resampling354 cleanup (bool): remove temporary files355 verbose (bool): display progress information356 interpolation (str): method to use for interpolating to the resmapled image357 358 Returns:359 (array or str): data or file name of resampled image360 Notes: 361 * resolutions are assumed to be given for the axes of the intrinsic 362 orientation of the data and reference as when viewed by matplotlib or ImageJ363 * orientation: permuation of 1,2,3 with potential sign, indicating which 364 axes map onto the reference axes, a negative sign indicates reversal 365 of that particular axes366 * only a minimal set of information to detremine the resampling parameter 367 has to be given, e.g. dataSizeSource and dataSizeSink368 """369 370 orientation = fixOrientation(orientation);371 372 if isinstance(dataSizeSink, str):373 dataSizeSink = io.dataSize(dataSizeSink);374 #orient actual resolutions onto reference resolution 375 dataSizeSource = io.dataSize(source);376 377 dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 378 resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);379 380 dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);381 382 #print dataSizeSource, dataSizeSink, resolutionSource, resolutionSink, dataSizeSinkI383 384 385 #rescale in x y in parallel386 if processingDirectory == None:387 processingDirectory = tempfile.mkdtemp(); 388 389 interpolation = fixInterpolation(interpolation);390 391 nZ = dataSizeSource[2];392 pool = multiprocessing.Pool(processes=processes);393 argdata = [];394 for i in range(nZ):395 argdata.append( (source, os.path.join(processingDirectory, 'resample_%04d.tif' % i), dataSizeSinkI, interpolation, i, nZ, verbose) ); 396 #print argdata[i]397 pool.map(_resampleXYParallel, argdata);398 399 #rescale in z400 fn = os.path.join(processingDirectory, 'resample_%04d.tif' % 0);401 data = io.readData(fn);402 zImage = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], nZ), dtype = data.dtype); 403 for i in range(nZ):404 if verbose and i % 10 == 0:405 print("resampleData; reading %d/%d" % (i, nZ));406 fn = os.path.join(processingDirectory, 'resample_%04d.tif' % i);407 zImage[:,:, i] = io.readData(fn);408 409 resampledData = numpy.zeros(dataSizeSinkI, dtype = zImage.dtype);410 for i in range(dataSizeSinkI[0]):411 if verbose and i % 25 == 0:412 print("resampleData: processing %d/%d" % (i, dataSizeSinkI[0]))413 #resampledImage[:, iImage ,:] = scipy.misc.imresize(zImage[:,iImage,:], [resizedZAxisSize, sagittalImageSize[1]] , interp = 'bilinear'); 414 #cv2.resize takes reverse order of sizes !415 resampledData[i ,:, :] = cv2.resize(zImage[i,:,:], (dataSizeSinkI[2], dataSizeSinkI[1]), interpolation = interpolation);416 #resampledData[i ,:, :] = cv2.resize(zImage[i,:, :], (dataSize[1], resizedZSize));417 418 #account for using (z,y,x) array representation -> (y,x,z)419 #resampledData = resampledData.transpose([1,2,0]);420 #resampledData = resampledData.transpose([2,1,0]);421 422 if cleanup:423 shutil.rmtree(processingDirectory);424 if not orientation is None:425 426 #reorient427 per = orientationToPermuation(orientation);428 resampledData = resampledData.transpose(per);429 430 #reverse orientation after permuting e.g. (-2,1) brings axis 2 to first axis and we can reorder there431 if orientation[0] < 0:432 resampledData = resampledData[::-1, :, :];433 if orientation[1] < 0:434 resampledData = resampledData[:, ::-1, :]; 435 if orientation[2] < 0:436 resampledData = resampledData[:, :, ::-1];437 438 #bring back from y,x,z to z,y,x439 #resampledImage = resampledImage.transpose([2,0,1]);440 if verbose:441 print("resampleData: resampled data size: " + str(resampledData.shape))442 443 if sink == []:444 if io.isFileExpression(source):445 sink = os.path.split(source);446 sink = os.path.join(sink[0], 'resample_\d{4}.tif');447 elif isinstance(source, str):448 sink = source + '_resample.tif';449 else:450 raise RuntimeError('resampleData: automatic sink naming not supported for non string source!');451 452 return io.writeData(sink, resampledData);453 454 455 456def resampleDataInverse(sink, source = None, dataSizeSource = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), 457 processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):458 """Resample data inversely to :func:`resampleData` routine459 460 Arguments:461 sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)462 source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)463 dataSizeSource (tuple or None): target size of the resampled image464 orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)465 resolutionSource (tuple): resolution of the source image (in length per pixel)466 resolutionSink (tuple): resolution of the resampled image (in length per pixel)467 processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created468 processes (int): number of processes to use for parallel resampling469 cleanup (bool): remove temporary files470 verbose (bool): display progress information471 interpolation (str): method to use for interpolating to the resmapled image472 473 Returns:474 (array or str): data or file name of resampled image475 Notes: 476 * resolutions are assumed to be given for the axes of the intrinsic 477 orientation of the data and reference as when viewed by matplotlib or ImageJ478 * orientation: permuation of 1,2,3 with potential sign, indicating which 479 axes map onto the reference axes, a negative sign indicates reversal 480 of that particular axes481 * only a minimal set of information to detremine the resampling parameter 482 has to be given, e.g. dataSizeSource and dataSizeSink483 """ 484 485 486 #orientation487 orientation = fixOrientation(orientation);488 489 #assume we can read data fully into memory490 resampledData = io.readData(sink);491 dataSizeSink = resampledData.shape;492 493 if isinstance(dataSizeSource, str):494 dataSizeSource = io.dataSize(dataSizeSource);495 dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 496 resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);497 #print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )498 499 dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);500 501 502 #flip axes back and permute inversely503 if not orientation is None:504 if orientation[0] < 0:505 resampledData = resampledData[::-1, :, :];506 if orientation[1] < 0:507 resampledData = resampledData[:, ::-1, :]; 508 if orientation[2] < 0:509 resampledData = resampledData[:, :, ::-1];510 511 #reorient512 peri = inverseOrientation(orientation);513 peri = orientationToPermuation(peri);514 resampledData = resampledData.transpose(peri);515 516 # upscale in z517 interpolation = fixInterpolation(interpolation);518 519 resampledDataXY = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]), dtype = resampledData.dtype); 520 521 for i in range(dataSizeSinkI[0]):522 if verbose and i % 25 == 0:523 print("resampleDataInverse: processing %d/%d" % (i, dataSizeSinkI[0]))524 #cv2.resize takes reverse order of sizes !525 resampledDataXY[i ,:, :] = cv2.resize(resampledData[i,:,:], (dataSizeSource[2], dataSizeSinkI[1]), interpolation = interpolation);526 # upscale x, y in parallel527 528 if io.isFileExpression(source):529 files = source;530 else:531 if processingDirectory == None:532 processingDirectory = tempfile.mkdtemp(); 533 files = os.path.join(sink[0], 'resample_\d{4}.tif');534 535 io.writeData(files, resampledDataXY);536 537 nZ = dataSizeSource[2];538 pool = multiprocessing.Pool(processes=processes);539 argdata = [];540 for i in range(nZ):541 argdata.append( (source, fl.fileExpressionToFileName(files, i), dataSizeSource, interpolation, i, nZ) ); 542 pool.map(_resampleXYParallel, argdata);543 544 if io.isFileExpression(source):545 return source;546 else:547 data = io.convertData(files, source);548 549 if cleanup:550 shutil.rmtree(processingDirectory);551 552 return data;553 554def resamplePoints(pointSource, pointSink = None, dataSizeSource = None, dataSizeSink = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), **args):555 """Resample Points to map from original data to the coordinates of the resampled image556 557 The resampling of points here corresponds to he resampling of an image in :func:`resampleData`558 559 Arguments:560 pointSource (str or array): image to be resampled561 pointSink (str or None): destination of resampled image562 orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)563 dataSizeSource (str, tuple or None): size of the data source564 dataSizeSink (str, tuple or None): target size of the resampled image565 resolutionSource (tuple): resolution of the source image (in length per pixel)566 resolutionSink (tuple): resolution of the resampled image (in length per pixel)567 568 Returns:569 (array or str): data or file name of resampled points570 Notes: 571 * resolutions are assumed to be given for the axes of the intrinsic 572 orientation of the data and reference as when viewed by matplotlib or ImageJ573 * orientation: permuation of 1,2,3 with potential sign, indicating which 574 axes map onto the reference axes, a negative sign indicates reversal 575 of that particular axes576 * only a minimal set of information to detremine the resampling parameter 577 has to be given, e.g. dataSizeSource and dataSizeSink578 """579 580 #fix (y,x,z) image array representation581 #resolutionSource, resolutionSink = self.fixResolutions(resolutionSource, resolutionSink);582 583 orientation = fixOrientation(orientation);584 #datasize of data source585 if isinstance(dataSizeSource, str):586 dataSizeSource = io.dataSize(dataSizeSource);587 588 dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 589 resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);590 points = io.readPoints(pointSource);591 dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);592 #resolutionSinkI = orientResolutionInverse(resolutionSink, orientation);593 594 #scaling factors595 scale = [float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3)];596 #print scale597 598 repoints = points.copy();599 for i in range(3): 600 repoints[:,i] = repoints[:,i] / scale[i];601 602 #permute for non trivial orientation603 if not orientation is None:604 per = orientationToPermuation(orientation);605 repoints = repoints[:,per];606 607 for i in range(3):608 if orientation[i] < 0:609 repoints[:,i] = dataSizeSink[i] - repoints[:,i];610 611 return io.writePoints(pointSink, repoints);612 613def resamplePointsInverse(pointSource, pointSink = None, dataSizeSource = None, dataSizeSink = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), **args):614 """Resample points from the coordinates of the resampled image to the original data615 The resampling of points here corresponds to he resampling of an image in :func:`resampleDataInverse`616 617 Arguments:618 pointSource (str or array): image to be resampled619 pointSink (str or None): destination of resampled image620 orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)621 dataSizeSource (str, tuple or None): size of the data source622 dataSizeSink (str, tuple or None): target size of the resampled image623 resolutionSource (tuple): resolution of the source image (in length per pixel)624 resolutionSink (tuple): resolution of the resampled image (in length per pixel)625 626 Returns:627 (array or str): data or file name of inversely resampled points628 Notes: 629 * resolutions are assumed to be given for the axes of the intrinsic 630 orientation of the data and reference as when viewed by matplotlib or ImageJ631 * orientation: permuation of 1,2,3 with potential sign, indicating which 632 axes map onto the reference axes, a negative sign indicates reversal 633 of that particular axes634 * only a minimal set of information to detremine the resampling parameter 635 has to be given, e.g. dataSizeSource and dataSizeSink636 """637 638 orientation = fixOrientation(orientation);639 640 #datasize of data source641 if isinstance(dataSizeSource, str):642 dataSizeSource = io.dataSize(dataSizeSource);643 644 dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 645 resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);646 647 points = io.readPoints(pointSource);648 649 dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);650 #resolutionSinkI = orientResolutionInverse(resolutionSink, orientation);651 652 #scaling factors653 scale = [float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3)];654 #print scale655 rpoints = points.copy(); 656 657 #invert axis inversion and permutations 658 if not orientation is None:659 #invert permuation660 iorientation = inverseOrientation(orientation);661 per = orientationToPermuation(iorientation);662 rpoints = rpoints[:,per];663 664 for i in range(3):665 if iorientation[i] < 0:666 rpoints[:,i] = dataSizeSink[i] - rpoints[:,i];667 668 #scale points669 for i in range(3): 670 rpoints[:,i] = rpoints[:,i] * scale[i]; 671 672 return io.writePoints(pointSink, rpoints);673def sagittalToCoronalData(source, sink = None):674 """Change from saggital to coronal orientation675 676 Arguments:677 source (str or array): source data to be reoriented678 sink (str or None): destination for reoriented image679 680 Returns:681 str or array: reoriented data682 """683 684 source = io.readData(source);685 d = source.ndim;686 if d < 3:687 raise RuntimeError('sagittalToCoronalData: 3d image required!');688 689 tp = range(d);690 tp[0:3] = [2,0,1];691 source = source.transpose(tp);692 source = source[::-1];693 #source = source[::-1,:,:];694 return io.writeData(sink, source);695def _test():696 """Tests for the Resampling Module"""697 import ClearMap.Alignment.Resampling as self698 imp.reload(self)699 from ClearMap.Settings import ClearMapPath as basedir 700 import iDISCO.IO.IO as io701 import os, numpy702 fn = os.path.join(basedir, 'Test/Data/OME/16-17-27_0_8X-s3-20HF_UltraII_C00_xyz-Table Z\d{4}.ome.tif');703 outfn = os.path.join(basedir, "Test/Data/Resampling/test.mhd")704 705 print("Making resampled stack " + outfn)706 print("source datasize %s" % str(io.dataSize(fn)));707 data = self.resampleData(fn, sink = None, resolutionSource = (1,1,1), orientation = (1,2,3), resolutionSink = (10,10,2));708 print(data.shape)709 io.writeData(outfn, data) 710 data = self.resampleData(fn, sink = None, dataSizeSink = (50,70,10), orientation = (1,2,3));711 print(data.shape)712 io.writeData(outfn, data) 713 dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = self.resampleDataSize(dataSizeSource = (100,200, 303), dataSizeSink = None, 714 resolutionSource = (1,1,1), resolutionSink = (5,5,5), orientation = (1,2,3));715 print(dataSizeSource, dataSizeSink, resolutionSource, resolutionSink)716 717 points = numpy.array([[0,0,0], [1,1,1], io.dataSize(fn)]);718 points = points.astype('float')719 pr = self.resamplePoints(points, dataSizeSource = fn, dataSizeSink = (50,70,10), orientation = (1,2,3))720 print(pr)721 pri = self.resamplePointsInverse(pr, dataSizeSource = fn, dataSizeSink = (50,70,10), orientation = (-1,2,3))722 print(pri)723 result = self.resampleDataInverse(outfn, os.path.join(basedir, 'Test/Data/OME/resample_\d{4}.ome.tif'), dataSizeSource = fn);724 print(result)725if __name__ == "__main__":...

Full Screen

Full Screen

CreateHaloFiles.py

Source:CreateHaloFiles.py Github

copy

Full Screen

1#!/usr/bin/env conda-env-py32import tangos3import os4import numpy as np5import h5py6h = 0.67769427832679697from tangos_halo_module.path import get_file_path, get_halo_snap_num8from tangos_halo_module.halos import get_survivors, get_zombies, get_host, get_survivor_IDs, get_zombie_IDs, blockPrint, enablePrint9from tangos_halo_module.halo_properties import track_halo_property, get_timesteps10def track_halo_property(simulation, key, tangos_halo=0, halo_path=0, halo_id=0, snap_num=0, resolution=1000):11 '''12 input params: 13 simulation: h148, h229, h242, h329: string14 key: pre-existing tangos halo property (eg. 'Mvir' or 'VXc'): string15 tangos_halo: a valid Tangos halo object16 halo_path: 0 or a complete tangos halo address: path or string object17 halo_id: halo id: string or numeric18 snap_num: 4 digit simulation snapshot number: string19 output params:20 tracked parameter: earliest --> latest snapshot: numpy array21 '''22 23 if tangos_halo:24 halo = tangos_halo25 elif halo_id and snap_num and simulation:26 #string tangos halo query from components27 if simulation == 'h148':28 halo = tangos.get_halo("snapshots/"+ str(simulation) +29 ".cosmo50PLK.3072g3HbwK1BH.00"+ str(snap_num) +"/"+ str(halo_id))30 else:31 halo = tangos.get_halo("snapshots/"+ str(simulation) +32 ".cosmo50PLK.3072gst5HbwK1BH.00"+ str(snap_num) +"/"+ str(halo_id))33 elif halo_path:34 halo = tangos.get_halo(str(halo))35 else:36 raise ValueError("Halo %r not found. You got this tho." % (halo))37 #edit .db path to match local address38 if resolution==1000:39 tangos.init_db("simulation_database/"+ str(simulation) +".db")40 elif resolution==100:41 tangos.init_db("/data/Sims/Databases/100Nptcls/"+ str(simulation) +".db")42 else:43 raise ValueError("Resolution not implemented yet.")44 45 # round all times in Gyr to 2 decimal places due to 46 # inconsistent float truncation between response to livecalculation query and direct database query47 halo_gyr = round(halo.timestep.time_gyr, 2)48 prog = np.flip(halo.calculate_for_progenitors(str(key))[0])49 desc = halo.calculate_for_descendants(str(key))[0][1:]50 track = np.array(np.concatenate((prog, desc), axis=None))51 len_track = len(track)52 timesteps = get_timesteps(simulation)[0] 53 timesteps = [round(t, 2) for t in timesteps] #in gyr since start of sim54 num_timesteps = len(timesteps)55 #len(tangos.get_simulation("snapshots").timesteps) #62 for h229, h242, h32956 57 if len_track == num_timesteps:58 return np.asarray(track)59 elif len_track > num_timesteps/2: #pad track with initial 0s60 return np.array(np.concatenate((np.zeros(num_timesteps - len_track), track), axis=None), dtype=float)61 else:62 # i = np.where(timesteps==halo_gyr)[0][0]63 i = timesteps.index(halo_gyr)64 new_track = np.zeros(num_timesteps)65 new_track[i+1:i+1+len(desc)] = desc66 new_track[i+1-len(prog):i+1] = prog67 return new_track 68 69def write_halo_file(halo, simulation, status, mode='add', resolution=100):70 import h5py71 h = 0.677694278326796972 73 if mode == 'write': # Write74 m = 'w'75 else: # Add76 m = 'a'77 path = get_file_path(tangos_halo=halo, simulation=simulation, status=status, halo_id=0, snap_num=0, resolution=resolution)78 with h5py.File(path, m) as f:79 d1 = f.create_dataset('time_|_Gyr', data = get_timesteps(simulation=simulation, resolution=resolution)[0])80 d2 = f.create_dataset('time_|_redshift', data = get_timesteps(simulation=simulation, resolution=resolution)[1])81 d3 = f.create_dataset('time_|_a', data = get_timesteps(simulation=simulation, resolution=resolution)[2])82# d4 = f.create_dataset('SFR_100Myr_|_Msol/yr', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='SFR_100Myr', resolution=resolution))83 d5 = f.create_dataset('Xc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Xc', resolution=resolution)*a/h)84 d6 = f.create_dataset('Yc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Yc', resolution=resolution)*a/h)85 d7 = f.create_dataset('Zc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Zc', resolution=resolution)*a/h)86 d8 = f.create_dataset('Rvir_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Rvir', resolution=resolution)*a/h)87 d9 = f.create_dataset('Mvir_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Mvir', resolution=resolution)*h)88 d10 = f.create_dataset('M_star_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='M_star', resolution=resolution)*h)89 d11 = f.create_dataset('VXc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VXc', resolution=resolution))90 d12 = f.create_dataset('VYc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VYc', resolution=resolution))91 d13 = f.create_dataset('VZc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VZc', resolution=resolution))92 d14 = f.create_dataset('n_gas', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_gas', resolution=resolution))93 d15 = f.create_dataset('n_star', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_star', resolution=resolution))94 d16 = f.create_dataset('n_dm', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_dm', resolution=resolution))95 d17 = f.create_dataset('M_gas_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='M_gas', resolution=resolution)*h)96 return97# Write data98for sim in ['h148', 'h229', 'h242', 'h329']:99 100 # Host101 print('Started Host: ' + str(sim))102 halo = get_host(simulation=sim, resolution=100)103 a = get_timesteps(simulation=sim, resolution=100)[2]104 write_halo_file(halo=halo, simulation=sim, status='Host', resolution=100, mode='write')105 print('Finished with Host: ' + str(sim))106 107 print('Started Survivors: ' + str(sim))108 survivors = get_survivors(simulation=sim, resolution=100)109 for sat in survivors:110 write_halo_file(halo=sat, simulation=sim, status='Survivor', resolution=100, mode='write')111 print('Finished with Survivors: ' + str(sim))112 113 print('Started Zombies: ' + str(sim))114 zombies = get_zombies(simulation=sim, resolution=100)115 for sat in zombies:116 write_halo_file(halo=sat, simulation=sim, status='Zombie', resolution=100, mode='write')...

Full Screen

Full Screen

pg_rec_losses.py

Source:pg_rec_losses.py Github

copy

Full Screen

...8 self._resolution = max_resolution9 self._stage = ProgGrowStageType.stab10 self._progress = 011 @abstractmethod12 def set_stage_resolution(self, stage, resolution):13 pass14 @abstractmethod15 def set_progress(self, progress):16 pass17 @abstractmethod18 def forward(self, x, y):19 pass20 @abstractmethod21 def set_reduction(self, reduction):22 pass23class PGPerceptualLoss(AbstractPGLoss):24 def __init__(self, max_resolution, weights_per_resolution,25 reduction='mean',26 use_smooth_pg=False,27 use_feature_normalization=False,28 use_L1_norm=False,29 use_relative_error=False):30 super(PGPerceptualLoss, self).__init__(max_resolution)31 self._max_resolution = max_resolution32 self._weights_per_resolution = weights_per_resolution33 self._use_smooth_pg = use_smooth_pg34 self._loss = PerceptualLoss(reduction=reduction,35 use_feature_normalization=use_feature_normalization,36 use_L1_norm=use_L1_norm,37 use_relative_error=use_relative_error)38 self._resolution = self._max_resolution39 self._stage = ProgGrowStageType.stab40 self._progress = 041 def set_stage_resolution(self, stage, resolution):42 self._stage = stage43 self._resolution = resolution44 self._progress = 045 def set_progress(self, progress):46 self._progress = progress47 def set_reduction(self, reduction):48 self._loss.reduction = reduction49 def forward(self, x, y):50 self._loss.set_new_weights(**self._weights_per_resolution[self._resolution])51 loss = self._loss(x, y)52 if self._use_smooth_pg:53 if self._stage == ProgGrowStageType.trns and self._progress < 1:54 prev_res = int(self._resolution / 2)55 self._loss.set_new_weights(**self._weights_per_resolution[prev_res])56 x = torch.nn.functional.upsample(x, scale_factor=0.5, mode='bilinear')57 y = torch.nn.functional.upsample(y, scale_factor=0.5, mode='bilinear')58 prev_loss = self._loss(x, y)59 loss = (1 - self._progress) * prev_loss + self._progress * loss60 return loss61class PGRelativePerceptualL1Loss(PGPerceptualLoss):62 def __init__(self, max_resolution, weights_per_resolution, reduction='mean', use_smooth_pg=False):63 super().__init__(64 max_resolution, weights_per_resolution,65 reduction=reduction,66 use_smooth_pg=use_smooth_pg,67 use_feature_normalization=True,68 use_L1_norm=True,69 use_relative_error=True)70class PGL2Loss(AbstractPGLoss):71 def __init__(self, max_resolution, reduction='mean'):72 super().__init__(max_resolution)73 self._loss = L2Loss(reduction=reduction)74 def set_stage_resolution(self, stage, resolution):75 pass76 def set_progress(self, progress):77 pass78 def set_reduction(self, reduction):79 self._loss.set_reduction(reduction)80 def forward(self, x, y):81 return self._loss(x, y)82class PGL1Loss(AbstractPGLoss):83 def __init__(self, max_resolution, reduction='mean'):84 super().__init__(max_resolution)85 self._loss = L1Loss(reduction=reduction)86 def set_stage_resolution(self, stage, resolution):87 pass88 def set_progress(self, progress):89 pass90 def set_reduction(self, reduction):91 self._loss.set_reduction(reduction)92 def forward(self, x, y):93 return self._loss(x, y)94class PGComposeLoss(AbstractPGLoss):95 def __init__(self, max_resolution, loss_1, loss_2):96 super().__init__(max_resolution)97 self.loss_1 = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_1['loss_type']]](98 max_resolution=max_resolution, **loss_1['loss_kwargs'])99 self.loss_1_weight = loss_1['loss_weight']100 self.loss_2 = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_2['loss_type']]](101 max_resolution=max_resolution, **loss_2['loss_kwargs'])102 self.loss_2_weight = loss_2['loss_weight']103 def set_stage_resolution(self, stage, resolution):104 self.loss_1.set_stage_resolution(stage, resolution)105 self.loss_2.set_stage_resolution(stage, resolution)106 def set_progress(self, progress):107 self.loss_1.set_progress(progress)108 self.loss_2.set_progress(progress)109 def forward(self, x, y):110 return self.loss_1_weight * self.loss_1(x, y) + self.loss_2_weight * self.loss_2(x, y)111 def set_reduction(self, reduction):112 self.loss_1.set_reduction(reduction)113 self.loss_2.set_reduction(reduction)114PG_RECONSTRUCTION_LOSSES = {115 ReconstructionLossType.perceptual: PGPerceptualLoss,116 ReconstructionLossType.relative_perceptual_L1: PGRelativePerceptualL1Loss,117 ReconstructionLossType.l1: PGL1Loss,118 ReconstructionLossType.l2: PGL2Loss,119 ReconstructionLossType.compose: PGComposeLoss...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run ATX automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful