0

I'm using the below code which is using Win32 calls to get a super fast file list of a directory. It is very quick but has a couple of limitations.

1) It doesn't handle very long paths. Any files below the Windows 260 character path limit will not be found. Potential Answer Below

2) This filelist will not enter shortcut folders like the one below.

enter image description here

Is there anyway around these limitations while still holding onto the speed of the operation?

private static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1);

[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
private static extern IntPtr FindFirstFileW(string lpFileName, out WIN32_FIND_DATAW lpFindFileData);

[DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
private static extern bool FindNextFile(IntPtr hFindFile, out WIN32_FIND_DATAW lpFindFileData);

[DllImport("kernel32.dll")]
private static extern bool FindClose(IntPtr hFindFile);

private static bool FindNextFilePInvokeRecursive(string path, out List<FileInformation> files,
    out List<DirectoryInformation> directories)
{
    var fileList = new List<FileInformation>();
    var directoryList = new List<DirectoryInformation>();
    var findHandle = INVALID_HANDLE_VALUE;
    //var info = new List<Tuple<string, DateTime>>();
    try
    {
        findHandle = FindFirstFileW(path + @"\*", out var findData);
        if (findHandle != INVALID_HANDLE_VALUE)
            do
            {
                // Skip current directory and parent directory symbols that are returned.
                if (findData.cFileName != "." && findData.cFileName != "..")
                {
                    var fullPath = path + @"\" + findData.cFileName;
                    // Check if this is a directory and not a symbolic link since symbolic links could lead to repeated files and folders as well as infinite loops.
                    if (findData.dwFileAttributes.HasFlag(FileAttributes.Directory) &&
                        !findData.dwFileAttributes.HasFlag(FileAttributes.ReparsePoint))
                    {
                        directoryList.Add(new DirectoryInformation {FullPath = fullPath});
                        var subDirectoryFileList = new List<FileInformation>();
                        var subDirectoryDirectoryList = new List<DirectoryInformation>();
                        if (FindNextFilePInvokeRecursive(fullPath, out subDirectoryFileList,
                            out subDirectoryDirectoryList))
                        {
                            fileList.AddRange(subDirectoryFileList);
                            directoryList.AddRange(subDirectoryDirectoryList);
                        }
                    }
                    else if (!findData.dwFileAttributes.HasFlag(FileAttributes.Directory))
                    {
                        fileList.Add(new FileInformation {FullPath = fullPath});
                    }
                }
            } while (FindNextFile(findHandle, out findData));
    }
    catch (Exception exception)
    {
        Console.WriteLine("Caught exception while trying to enumerate a directory. {0}", exception);
        if (findHandle != INVALID_HANDLE_VALUE) FindClose(findHandle);
        files = null;
        directories = null;
        return false;
    }
    if (findHandle != INVALID_HANDLE_VALUE) FindClose(findHandle);
    files = fileList;
    directories = directoryList;
    return true;
}

private static List<FileInformation> FindNextFilePInvokeRecursiveParalleled(string path)
{
    List<FileInformation> files;
    var fileList = new List<FileInformation>();
    var fileListLock = new object();
    var directoryList = new List<DirectoryInformation>();
    var directoryListLock = new object();
    var findHandle = INVALID_HANDLE_VALUE;
    //var info = new List<Tuple<string, DateTime>>();
    try
    {
        path = path.EndsWith(@"\") ? path : path + @"\";
        findHandle = FindFirstFileW(path + @"*", out var findData);
        if (findHandle != INVALID_HANDLE_VALUE)
        {
            do
            {
                // Skip current directory and parent directory symbols that are returned.
                if (findData.cFileName != "." && findData.cFileName != "..")
                {
                    var fullPath = path + findData.cFileName;
                    // Check if this is a directory and not a symbolic link since symbolic links could lead to repeated files and folders as well as infinite loops.
                    if (findData.dwFileAttributes.HasFlag(FileAttributes.Directory) &&
                        !findData.dwFileAttributes.HasFlag(FileAttributes.ReparsePoint))
                        directoryList.Add(new DirectoryInformation {FullPath = fullPath});
                    else if (!findData.dwFileAttributes.HasFlag(FileAttributes.Directory))
                        fileList.Add(new FileInformation {FullPath = fullPath});
                }
            } while (FindNextFile(findHandle, out findData));
            directoryList.AsParallel().ForAll(x =>
            {
                var subDirectoryFileList = new List<FileInformation>();
                var subDirectoryDirectoryList = new List<DirectoryInformation>();
                if (FindNextFilePInvokeRecursive(x.FullPath, out subDirectoryFileList,
                    out subDirectoryDirectoryList))
                {
                    lock (fileListLock)
                    {
                        fileList.AddRange(subDirectoryFileList);
                    }
                    lock (directoryListLock)
                    {
                        directoryList.AddRange(subDirectoryDirectoryList);
                    }
                }
            });
        }
    }
    catch (Exception exception)
    {
        Console.WriteLine("Caught exception while trying to enumerate a directory. {0}", exception);
        if (findHandle != INVALID_HANDLE_VALUE) FindClose(findHandle);
        files = null;
        //directories = null;
        return null;
    }
    if (findHandle != INVALID_HANDLE_VALUE) FindClose(findHandle);
    files = fileList;
    //directories = directoryList;
    return files;
}

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
private struct WIN32_FIND_DATAW
{
    public readonly FileAttributes dwFileAttributes;
    internal readonly FILETIME ftCreationTime;
    internal readonly FILETIME ftLastAccessTime;
    internal readonly FILETIME ftLastWriteTime;
    private readonly int nFileSizeHigh;
    private readonly int nFileSizeLow;
    private readonly int dwReserved0;
    private readonly int dwReserved1;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 260)] public readonly string cFileName;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 14)] public readonly string cAlternateFileName;
}

private class FileInformation
{
    public string FullPath;
    //public DateTime LastWriteTime;
}

private class DirectoryInformation
{
    public string FullPath;
    //public DateTime LastWriteTime;
}

Solution to 1 By appending the windows long path prefix (\\?\) onto the path that the user inputs, it appears to search much further down the file path! Haven't fully tested this yet but working in initial tests!! Link

Now I think I just need a solution to question 2.

5
  • paths over the 260 limit i thought were OS limit not code limit.. and the linked stuff are links so show as files.. you could probably code something to follow them.. I was more interested in whether this is really that much faster than enumerate files etc.
    – BugFinder
    Nov 24, 2017 at 11:08
  • @BugFinder Here is the link to the original. Give it a quick test too, I was surprised at how much speed gain I got. stackoverflow.com/questions/26321366/…
    – windowsgm
    Nov 24, 2017 at 11:22
  • Ok - gonna look into that cos I used a queue system and parallel threads so to be honest Id be surprised if there was a huge difference but you sure got me interested
    – BugFinder
    Nov 24, 2017 at 11:25
  • I cant use your code it wont compile "out var findData" .. but I will poke at it and see what falls out
    – BugFinder
    Nov 24, 2017 at 11:56
  • Oh just change to out findData and declare var findData further up.
    – windowsgm
    Nov 24, 2017 at 12:02

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.