loader.py 文件源码

def get_spiders_classes(entry_point_name=None):
    discovered = dict()
    if not entry_point_name:
        entry_point_name = __name__.split(".")[0]

    def load_module(module_name):
        module = inspect.importlib.import_module(module_name)
        for importer, modname, ispkg in pkgutil.walk_packages(path=module.__path__):
            mod = inspect.importlib.import_module(module.__name__ + "." + modname)
            if ispkg:
                load_module(mod.__name__)
            else:
                for name, obj in inspect.getmembers(mod):
                    if is_spider(obj) and mod.__name__ == obj.__module__:
                        try:
                            sn = getattr(obj, "name", None)
                            if sn in discovered:
                                raise Exception("Duplicate spider '{}': {} and {}".format(sn, obj, discovered[sn]))
                            discovered[sn] = obj
                        except AttributeError:
                            pass

    load_module(module_name=entry_point_name)
    return discovered