banner
biuaxia

biuaxia

"万物皆有裂痕，那是光进来的地方。"
github
bilibili
tg_channel
首頁封存

Golang遍历文件及文件夹且计算文件MD5

302
AI 生成的摘要
這段程式碼是使用Golang遍歷文件和文件夾，並計算文件的MD5值。它定義了一個Md5sum函數來計算文件的MD5值，並定義了一個Traverse函數來遍歷目錄並計算MD5值。它還定義了一個測試函數TestTraverse來測試Traverse函數。運行後會生成一個md5sums.txt文件，其中包含了文件的路徑和MD5值。

title: Golang 遍历文件及文件夹且计算文件 MD5
date: 2021-06-22 11:04:03
toc: true
category:

  • Golang
    tags:
  • Go
  • Golang
  • 遍历
  • 文件夹
  • 子文件
  • 计算
  • 文件
  • MD5
  • TODO

2023-4-18 10:58:46

util_file.go 代码如下：

package util

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// Md5sum 计算文件 MD5 值
func Md5sum(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return ""
		}
		return ""
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return ""
	}
	return hex.EncodeToString(h.Sum(nil))
}

var firstDir string

// Traverse 遍历目录并计算 MD5
func Traverse(dir string) {
	// 遍历目录
	files, err := os.ReadDir(dir)
	if err != nil {
		panic(err)
	}
	subDirs := make([]string, 0)
	subFiles := make([]string, 0)

	// 找到子目录
	for _, f := range files {
		if f.IsDir() {
			subDirs = append(subDirs, f.Name())
		} else {
			subFiles = append(subFiles, f.Name())
		}
	}

	// 第一次调用时创建md5sums.txt文件
	if firstDir == "" {
		firstDir = dir
	}
	f, err := os.OpenFile(filepath.Join(firstDir, "md5sums.txt"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		panic(err)
	}

	if f != nil {
		defer f.Close()
	}

	done := make(chan struct{})
	go func() {
		// 遍历子目录
		for _, subDir := range subDirs {
			dirPath := filepath.Join(dir, subDir)
			Traverse(dirPath)
		}
		close(done)
	}()

	go func() {
		// 计算文件 MD5
		for _, file := range subFiles {
			md5FilePath := filepath.Join(dir, file)
			md5sum := Md5sum(md5FilePath)
			if md5sum != "" {
				if f != nil {
					_, err := fmt.Fprintf(f, "%s  %s\n", md5FilePath, md5sum)
					if err != nil {
						f, err = os.OpenFile(filepath.Join(firstDir, "md5sums.txt"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
						if err != nil {
							panic(err)
						}
						defer f.Close()
						_, err = fmt.Fprintf(f, "%s  %s\n", md5FilePath, md5sum)
						if err != nil {
							panic(err)
						}
					}
				}
			}
		}
		<-done
	}()

	<-done
}

util_file_test.go 代码如下：

package util

import (
	"testing"
)

func TestTraverse(t *testing.T) {
	strings := []string{
		"D:\\Workspace\\Project\\tank",
		"D:\\Workspace\\Project\\recipes",
	}
	for _, tt := range strings {
		Traverse(tt)
		firstDir = ""
	}
}

每次调用完 Traverse 方法记得将 firstDir 置为 ""，运行后将会得到 md5sums.txt 文件，内容如下：

D:\Workspace\Project\tank\.gitignore  49fc96a8f937b2f0e17ef26a2c3b5da2
D:\Workspace\Project\tank\CHANGELOG  aac6fc7d43b53c94e9d2fbc1ecaccc8b
D:\Workspace\Project\tank\.git\FETCH_HEAD  f42e34d4aa0b4539e4e0f1c4d812a789
D:\Workspace\Project\tank\Dockerfile  be292ab456a0b6f3a6d97dff28fd999b
D:\Workspace\Project\tank\.git\HEAD  4cf2d64e44205fe628ddd534e1151b58
D:\Workspace\Project\tank\LICENSE  7ada4a1c6456fbe950cf08d5d82a2c05
D:\Workspace\Project\tank\.git\hooks\applypatch-msg.sample  ce562e08d8098926a3862fc6e7905199
D:\Workspace\Project\tank\.git\config  a93576437c6e4c8d791a49a8a38a6d53
D:\Workspace\Project\tank\README.md  6a04e351df08269daa708ddee0b47d03
D:\Workspace\Project\tank\.git\description  a0a7c3fff21f2aea3cfa1d0316dd816c
D:\Workspace\Project\tank\SECURITY.md  b97f0ae5c5e07ecee29db97d36e231d6
D:\Workspace\Project\tank\.git\info\exclude  036208b4a1ab4a235d75c181e685e5a3
D:\Workspace\Project\tank\.git\hooks\commit-msg.sample  579a3c1e12a1e74a98169175fb913012
D:\Workspace\Project\tank\go.mod  633f20ff9d99ad09001926fdef82199f
D:\Workspace\Project\tank\.git\index  ea9612a5425e5c79b67aaca64f67336d
D:\Workspace\Project\tank\.git\hooks\fsmonitor-watchman.sample  a0b2633a2c8e97501610bd3f73da66fc
D:\Workspace\Project\tank\go.sum  33383355cd8477ac436c892c3347e81b
D:\Workspace\Project\tank\main.go  6aa0980ae206e9ad3396095b3cf36d7f
D:\Workspace\Project\tank\.git\hooks\post-update.sample  2b7ea5cee3c49ff53d41e00785eb974c
D:\Workspace\Project\tank\.git\logs\HEAD  7a96c79d1f5c016b99a1c800092a81a3
D:\Workspace\Project\tank\.git\packed-refs  d373a80b5aaf8519f93b9ec52a2c6fee
D:\Workspace\Project\tank\.git\hooks\pre-applypatch.sample  054f9ffb8bfe04a599751cc757226dda
D:\Workspace\Project\tank\.git\logs\refs\heads\master  7a96c79d1f5c016b99a1c800092a81a3
D:\Workspace\Project\tank\.git\hooks\pre-commit.sample  305eadbbcd6f6d2567e033ad12aabbc4
D:\Workspace\Project\tank\.git\hooks\pre-merge-commit.sample  39cb268e2a85d436b9eb6f47614c3cbc
D:\Workspace\Project\tank\.git\logs\refs\remotes\origin\HEAD  7a96c79d1f5c016b99a1c800092a81a3
D:\Workspace\Project\tank\.git\hooks\pre-push.sample  2c642152299a94e05ea26eae11993b13
D:\Workspace\Project\tank\.git\hooks\pre-rebase.sample  56e45f2bcbc8226d2b4200f7c46371bf
D:\Workspace\Project\tank\.git\hooks\pre-receive.sample  2ad18ec82c20af7b5926ed9cea6aeedd
D:\Workspace\Project\tank\.git\hooks\prepare-commit-msg.sample  2b5c047bdb474555e1787db32b2d2fc5
D:\Workspace\Project\tank\.git\hooks\push-to-checkout.sample  c7ab00c7784efeadad3ae9b228d4b4db
D:\Workspace\Project\tank\.git\hooks\update.sample  647ae13c682f7827c22f5fc08a03674e
D:\Workspace\Project\tank\.git\objects\pack\pack-da1bb5783cc0c2b96e3487d1b53aeff29dc1b8fe.idx  abb79a9f95068aea661df09e4b992978
D:\Workspace\Project\tank\.git\refs\heads\master  0c07035bb5e03adc1fcef252a8a8989e
D:\Workspace\Project\tank\.git\refs\remotes\origin\HEAD  73a00957034783b7b5c8294c54cd3e12
D:\Workspace\Project\tank\.idea\.gitignore  c38cf56f115e0ff9f543292d96369362
D:\Workspace\Project\tank\.idea\modules.xml  1201756a21ff0cb151592c0b1ae7a88e
D:\Workspace\Project\tank\.idea\tank.iml  4e17da5ade7136eab5a48dcc6e2766bc
D:\Workspace\Project\tank\.idea\vcs.xml  74d3a64f52848d5e8db631b1685e58c8
D:\Workspace\Project\tank\build\conf\tank.json  f55032366878a69a92bd300b7307849e
D:\Workspace\Project\tank\.idea\workspace.xml  9bc3f9591c6d2817852f6b460c88c7dd
D:\Workspace\Project\tank\build\doc\sql\schema-3.1.0.sql  1d5889383091fda14ac1b484f31407bf
D:\Workspace\Project\tank\build\doc\img\dingding.jpg  f0d9624f2c0e2425fac91302ede56c70
D:\Workspace\Project\tank\build\html\asset-manifest.json  579aca455fc66af5e9b6e91b6bd36f19
D:\Workspace\Project\tank\build\html\favicon.ico  b877eb456647ebce0f190eee1bbf7c38
D:\Workspace\Project\tank\build\doc\img\jb_beam.png  e521a96f220daabd14d6316a2a53574d
D:\Workspace\Project\tank\build\html\index.html  58a97885555a967f61be84aef9bf5be2
D:\Workspace\Project\tank\build\doc\img\login.png  b937c8971eb013d484984937011fa1be
D:\Workspace\Project\tank\build\doc\img\logo.png  21458adb4d8cc0f1f6b4ee85b5a10f91
D:\Workspace\Project\tank\build\html\static\media\archive.684c1e42f233aa9d53a8910d4fee091e.svg  77d78eb7f0a09aefee1f95f08411be46
D:\Workspace\Project\tank\build\html\static\media\audio.e7112b210bfaa8d0a2ab69f74a66eea4.svg  30a1ea0240c7d661749676da6b67f88e
D:\Workspace\Project\tank\build\doc\img\matters.png  e89c86237e2a65364566a95beff8fece
D:\Workspace\Project\tank\build\html\static\css\main.b03c51fb.css  4dd38d8fb3d78f529102f2aded376bbb
D:\Workspace\Project\tank\build\html\static\media\default-skin.f64c3af3d0d25b9e4e00.svg  b257fa9c5ac8c515ac4d77a667ce2943
D:\Workspace\Project\tank\build\pack\build.bat  ead8dee4dc8e14f6cecddd8ade3481e3
D:\Workspace\Project\tank\build\html\static\media\doc.01619bc74278a86447e699223663ad64.svg  1e70a87cbcfdd0396f2108aa45daf0df
D:\Workspace\Project\tank\build\pack\build.sh  d59542f1589456fb032feb8c9a561ce3
D:\Workspace\Project\tank\build\service\shutdown.sh  49be7f9e77de83fc95f095bb4846b779
D:\Workspace\Project\tank\build\service\startup.sh  b96ea7c0cf1b53ed018a216296903eac
D:\Workspace\Project\tank\build\html\static\media\empty.20ceb38d310075aa4c3cdaae210afb65.svg  babf4d0801025493697c945389bb2e68
D:\Workspace\Project\tank\build\doc\img\mobile.png  559c1603691d29dab8dd0aff989f7cc7
D:\Workspace\Project\tank\build\service\tank.service  2f7ffb530ffca63b446a48cab68e829f
D:\Workspace\Project\tank\build\html\static\media\file.074f42545edfc31c849bdddb013ad8b2.svg  fd3e15f8a714653c3005a2773cd00b15
D:\Workspace\Project\tank\code\core\application.go  56a72aae86753c102e55df7ee853f491
D:\Workspace\Project\tank\code\core\bean.go  e61a11f80de35dd18244406057bfdde6
D:\Workspace\Project\tank\build\html\static\media\folder.0bdd7430280c98bf4970b6af5a061c2c.svg  f8d1b5009857e24890a42c866bbfe925
D:\Workspace\Project\tank\build\doc\img\tank0.png  6faf822f8ea5fcd6eb2058c5bb16979c
D:\Workspace\Project\tank\code\core\config.go  82ca84d9d00524869ed07bcb2962c819
D:\Workspace\Project\tank\build\html\static\media\image.7660bae05269b3c8f2cdf669f04aef3b.svg  06f036cece1ba8775007f41d6d22933c
D:\Workspace\Project\tank\code\core\context.go  45c5229bd7280d55a7255bd0f04a3ae9
D:\Workspace\Project\tank\code\core\controller.go  00f43aef2e5cc91ad101009ab71497e9
D:\Workspace\Project\tank\code\core\global.go  8beb1b3d9944b29cdaa062e1da952898
D:\Workspace\Project\tank\build\doc\img\tank1.png  b4874671b95ed486c50fef665a632aac
D:\Workspace\Project\tank\code\core\handler.go  400a3ede0795860897af7bb29bf19c84
D:\Workspace\Project\tank\code\core\logger.go  58cb440c0d808416517db90b267249b9
D:\Workspace\Project\tank\build\html\static\media\logo.847e54ef7fb4b744fad4.png  21458adb4d8cc0f1f6b4ee85b5a10f91
D:\Workspace\Project\tank\build\html\static\media\pdf.eba1e4d316279fc69682dabae6fe6fa4.svg  c256fdcbab556414dd550c6c5c5e568d
D:\Workspace\Project\tank\build\doc\img\tank2.png  c48dd237ff6b20b4bd7da018fd9f05a4
D:\Workspace\Project\tank\build\html\static\media\ppt.7dc0cfc7fdf0e6f66fb94c81fb34575f.svg  9ec3e0182feb8729760dad76c4e3378d
D:\Workspace\Project\tank\code\rest\alien_controller.go  3d7e1617e951b061b194cb225ac8786e
D:\Workspace\Project\tank\code\rest\alien_service.go  105140018be0a3d2b0dfa9084fa882cd
D:\Workspace\Project\tank\build\html\static\media\psd.bf2ac5411c0132f292787f45855934f2.svg  be6691be501376640a7c1e1dd678b2a8
D:\Workspace\Project\tank\code\rest\base_bean.go  5cc49e1a3f839306e631e5bc156d2a90
D:\Workspace\Project\tank\build\html\static\css\main.b03c51fb.css.map  86831daa160f415f62f10344abaf3a62
D:\Workspace\Project\tank\code\support\tank_application.go  f0664a9c8d323a13fcf055c57fee8bea
D:\Workspace\Project\tank\build\html\static\media\text.8000463985257af8534ee29db23e1006.svg  49dc13ae6f6d73ab9bdb3969eee980fa
D:\Workspace\Project\tank\code\support\tank_config.go  61ff9ae0fac14361b97f3817b8bd7d8d
D:\Workspace\Project\tank\code\test\cron_test.go  7abb091d0291e68b7234386749e09ed4
D:\Workspace\Project\tank\code\rest\base_controller.go  da97f9999930d7c02b93e880c50e51b7
D:\Workspace\Project\tank\code\rest\base_dao.go  c3240865f64c1cf05bfdea8677b6aaac
D:\Workspace\Project\tank\build\html\static\media\video.adecb1d58a645ca8d1b1bffad4dbe777.svg  4387235d573fc92ce361822a3306e532
D:\Workspace\Project\tank\code\test\dav_test.go  026d66fb401e2b4b56f417001ba9592f
D:\Workspace\Project\tank\code\support\tank_context.go  3db15d8780e8f99650d417329d825471
D:\Workspace\Project\tank\code\rest\base_model.go  3fe82ffc844c59ea8777597345757a56
D:\Workspace\Project\tank\build\html\static\media\xls.02895077484083ddc5346ab720cd8d1b.svg  ef5ba31bd69bdb3c3aec2db584781e53
D:\Workspace\Project\tank\code\test\main_test.go  ee1900719a6287f6e59240801ae2c082
D:\Workspace\Project\tank\code\tool\builder\sql_builder.go  38c898f5c4a4115ddeec40804b755dd2
D:\Workspace\Project\tank\code\support\tank_logger.go  72443b6f10adce303a183cd873358887
D:\Workspace\Project\tank\code\test\regex_test.go  6c870912841c7a6e6b6b9dc2435f899a
D:\Workspace\Project\tank\code\rest\bridge_dao.go  145c5a488eb0b807e0b8c2d9d9505c15
D:\Workspace\Project\tank\code\support\tank_router.go  600c1e14d4c7b1595f35ca142a08d5f2
D:\Workspace\Project\tank\code\rest\bridge_model.go  ff5bb18ea913e56056016e0a9ff3da4e
D:\Workspace\Project\tank\code\rest\bridge_service.go  f25d43999464fd11d4368361d111ff32
D:\Workspace\Project\tank\build\html\static\js\main.b94b1947.js  296b8fb99f71841380aa6d39d4ee1ac8
D:\Workspace\Project\tank\code\tool\cache\cache.go  cb22bf27dae4519b86cbb870af2069d7
D:\Workspace\Project\tank\code\rest\dashboard_controller.go  4dff797e27e5cb50bad4478d0fe3d546
D:\Workspace\Project\tank\build\html\static\js\main.b94b1947.js.LICENSE.txt  fa06b6e6cc838461959613646cdb63e1
D:\Workspace\Project\tank\code\tool\dav\prop.go  057d6853f03d02c78a4527839b605b0b
D:\Workspace\Project\tank\code\rest\dashboard_dao.go  11511ea8f0fa6cc584961ce4f3cbf62a
D:\Workspace\Project\tank\code\tool\dav\xml\marshal.go  6f21231692ccbbdd82ad7613bbb7ff0e
D:\Workspace\Project\tank\code\rest\dashboard_model.go  5afbe41db5b625b810e74665633201c6
D:\Workspace\Project\tank\code\tool\download\download.go  f0b296033489c43fa46f8d9790650195
D:\Workspace\Project\tank\code\tool\i18n\i18n.go  2107e89333a2ec94eb5905fa23b7b997
D:\Workspace\Project\tank\code\rest\dashboard_service.go  f8950c81c3798436d786b0c22668301f
D:\Workspace\Project\tank\code\tool\dav\xml\read.go  810e2f3526d47dd896f6a2497db6e60d
D:\Workspace\Project\tank\code\tool\third\gorm_helper.go  e283e9ad6f4ba7e29194a17a0afb246f
D:\Workspace\Project\tank\code\rest\dav_controller.go  9748f516a3c25292d6a92c4a11774ae3
D:\Workspace\Project\tank\code\tool\result\web_result.go  c67a8d9a6387747a50fe781885e34ea0
D:\Workspace\Project\tank\code\tool\dav\xml\typeinfo.go  f49d26205571fef418e96d4ec7155206
D:\Workspace\Project\tank\code\tool\util\util_cron.go  d6041d978925a24483417d57ae43a013
D:\Workspace\Project\tank\code\tool\util\util_encode.go  28addf000b3645d356be4dec25de8fe0
D:\Workspace\Project\tank\code\rest\dav_model.go  4d8bd025bbc691e5b85e720b7564978d
D:\Workspace\Project\tank\code\tool\uuid\example_test.go  fed13256b8f33a18b2200eac4f40da70
D:\Workspace\Project\tank\code\tool\util\util_env.go  ce8da9d0ce13f673aa5cfa7f091fd3ee
D:\Workspace\Project\tank\code\rest\dav_service.go  c6b8a4333407d3bb4a924b0115c7a7ea
D:\Workspace\Project\tank\code\tool\dav\xml\xml.go  c1a5eccf3b23017fec1b729823f2857f
D:\Workspace\Project\tank\code\tool\uuid\uuid.go  b5cb9f4ceedd658dc5783dd38ed16d3e
D:\Workspace\Project\tank\code\tool\util\util_file.go  bb3603d3ae00a9c8662f1fb33e5b6656
D:\Workspace\Project\tank\code\rest\download_token_dao.go  3e4ab3383f3852dd809d93d21000dd1e
D:\Workspace\Project\tank\code\tool\uuid\uuid_test.go  01d7d8be9dc952832e258309ea077f3b
D:\Workspace\Project\tank\code\tool\util\util_mime.go  6f45d68c3384f057ecf775e5db74b75a
D:\Workspace\Project\tank\code\tool\webdav\file.go  9fbfad90449993cd0aa9b3e31ea4accb
D:\Workspace\Project\tank\code\rest\download_token_model.go  e7990f279c6074ea43be2e1e7159f352
D:\Workspace\Project\tank\code\tool\util\util_network.go  145f4c79c6127a7fe7301b2e13abaa20
D:\Workspace\Project\tank\code\rest\footprint_controller.go  922336f6a629fe42cf1ff79472994834
D:\Workspace\Project\tank\code\tool\webdav\file_test.go  ca35d4a0a1e485071f464bbca43a5db2
D:\Workspace\Project\tank\code\tool\util\util_string.go  644d70162016497596269fa144d1a771
D:\Workspace\Project\tank\code\rest\footprint_dao.go  786e362ef15bb8b2db8b66c1aa65094e
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\README  d2ea35435e6c409f0bfb14120b8e4123
D:\Workspace\Project\tank\code\tool\util\util_time.go  3b6f8c1b97d1eaa64150f136e9f17f5f
D:\Workspace\Project\tank\code\tool\util\util_validation.go  dcedd3cc89e9ee9167717689e6617ec9
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\atom_test.go  3531cf62a00e60517c50143db83df99e
D:\Workspace\Project\tank\code\rest\footprint_model.go  f7f4ddaa32b4a9c6a000d7f607b7eb83
D:\Workspace\Project\tank\code\tool\util\util_zip.go  1a6a51aeae944b24261eb14eb3220468
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\example_test.go  97ac04a4cb5b4f7fa52da7a3ee90f599
D:\Workspace\Project\tank\code\rest\footprint_service.go  c6fd77d5fe6ef73a3f11630ab78e9f10
D:\Workspace\Project\tank\code\rest\image_cache_controller.go  ebd6a7a120bc629bb4f57a822880cdc5
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\marshal.go  6f21231692ccbbdd82ad7613bbb7ff0e
D:\Workspace\Project\tank\code\rest\image_cache_dao.go  64042f9beceb56b72a8f362163bbfc5b
D:\Workspace\Project\tank\code\rest\image_cache_model.go  115d11e2b192b2c66144be1612a12212
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\marshal_test.go  47018de89271131915d31051eaa008cb
D:\Workspace\Project\tank\code\rest\image_cache_service.go  036c16fc04431f88500cc73a9b58e7e4
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\read.go  810e2f3526d47dd896f6a2497db6e60d
D:\Workspace\Project\tank\code\rest\install_controller.go  2b80d6761355495761b7827a997da59f
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\read_test.go  40e71534d582ba706b5cffdc131dbd3a
D:\Workspace\Project\tank\code\rest\install_model.go  1028764ca5db8d44ed15382e9d6dba30
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\typeinfo.go  f49d26205571fef418e96d4ec7155206
D:\Workspace\Project\tank\code\rest\matter_controller.go  59338e8dbd267f6a56e35bc589f7f4f7
D:\Workspace\Project\tank\code\rest\matter_dao.go  86fffdc193720b5dedbfed5bd200cfbb
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\xml.go  c1a5eccf3b23017fec1b729823f2857f
D:\Workspace\Project\tank\code\tool\webdav\internal\xml\xml_test.go  b9ff8d72b4a1f9fe44f45d31ed4ab6fc
D:\Workspace\Project\tank\code\rest\matter_model.go  e67268f3ebc574e945a6835711d628c4
D:\Workspace\Project\tank\code\rest\matter_service.go  411ecbeb5c936410c082c18b87ae88ee
D:\Workspace\Project\tank\code\rest\preference_controller.go  ae97f1719449feab5756216c809e2da0
D:\Workspace\Project\tank\code\rest\preference_dao.go  4635d9a166bdc748c5d73f8c70497221
D:\Workspace\Project\tank\code\rest\preference_model.go  bb4f6a088f3665eebd40f60206e7866f
D:\Workspace\Project\tank\code\rest\preference_service.go  99331aa2a973247d00dfe845fee613ad
D:\Workspace\Project\tank\code\rest\session_dao.go  d05e8b32e16680e0940a5a0813a8131f
D:\Workspace\Project\tank\code\rest\session_model.go  f96a3d337847389badeff9336e9435dd
D:\Workspace\Project\tank\code\rest\session_service.go  fd5b721cd2cc71c74216c328afeceb28
D:\Workspace\Project\tank\code\rest\share_controller.go  2e75ea5f956cf57224b00269d3077af5
D:\Workspace\Project\tank\code\rest\share_dao.go  c8c06d573d0969f8141132507940a8d2
D:\Workspace\Project\tank\code\rest\share_model.go  4331a49721e0d4aa483a84beaa6014fb
D:\Workspace\Project\tank\code\rest\share_service.go  2689b62a6bca666fdf2626d751ccdce7
D:\Workspace\Project\tank\code\rest\task_service.go  583eecfe9e2a9642d5add0df13f4f90f
D:\Workspace\Project\tank\code\rest\upload_token_dao.go  a424cf6fcff75746defa57218b80d14a
D:\Workspace\Project\tank\code\rest\upload_token_model.go  32684782791c81b3861f5b849a5cb95a
D:\Workspace\Project\tank\code\rest\user_controller.go  f59c53094a6425392ec9900c449b145f
D:\Workspace\Project\tank\code\rest\user_dao.go  854bd4552474e22afa7ba30047aed5d1
D:\Workspace\Project\tank\code\rest\user_model.go  ebe2f447c944b611d04e6d543bb53612
D:\Workspace\Project\tank\code\rest\user_service.go  6edba6a1e01c62d41dbe1d00927c2f20
D:\Workspace\Project\tank\build\html\static\js\main.b94b1947.js.map  d054db641badfc68a620cf3b5f0db4f9
D:\Workspace\Project\tank\.git\objects\pack\pack-da1bb5783cc0c2b96e3487d1b53aeff29dc1b8fe.pack  1f769c77b88461967111b739e399adfe

2022 年 6 月 30 日 14 点 06 更新代码

package main

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
)

/*
1. 读取文件列表
2. 并发计算文件md5
3. 等待计算完毕将数据打印
*/
func main() {
	startTime := time.Now()

	var wg sync.WaitGroup

	workers, err := readDirAllFiles("C:\\Users\\Administrator\\Downloads\\SM\\2022\\01\\01", &wg)
	// workers, err := readDirAllFiles("C:\\ProgramData\\Microsoft\\Windows\\Start Menu\\Programs\\Windsys 工具箱", &wg)
	// workers, err := readDirAllFiles("C:\\Users\\Administrator\\Downloads\\.accelerate", &wg)
	// workers, err := readDirAllFiles("C:\\Users\\Administrator\\Downloads", &wg)
	if err != nil {
		panic(err)
	}

	var result []FileInfo
	for _, v := range workers {
		result = append(result, <-v.out)
	}

	fmt.Println(len(result), "总耗时", time.Since(startTime))

	wg.Wait()
}

func createWorker(id int, wg *sync.WaitGroup) worker {
	w := worker{
		in:   make(chan FileReceiveInfo),
		out:  make(chan FileInfo),
		done: wg.Done,
	}
	go doWork(id, w)
	return w
}

func doWork(id int, w worker) {
	defer w.done()
	receiveInfo := <-w.in
	fmt.Printf("No.%-7d worker receive path: %v, size: %v\n", id, receiveInfo.FilePath, receiveInfo.FileSize)

	startMd5CalcTime := time.Now()
	fileMd5, err := MD5sum(receiveInfo.FilePath)
	fmt.Printf("No.%-7d worker time: %v\n", id, time.Since(startMd5CalcTime))
	if err != nil {
		fmt.Printf("file [%s] has error: %v\n", receiveInfo.FilePath, err)
	}
	w.out <- FileInfo{
		FilePath: receiveInfo.FilePath,
		FileMd5:  fileMd5,
	}
}

type worker struct {
	in   chan FileReceiveInfo
	out  chan FileInfo
	done func()
}

type FileReceiveInfo struct {
	FilePath string
	FileSize int64
}

type FileInfo struct {
	FilePath, FileMd5 string
}

// 读取指定目录下的所有文件
func readDirAllFiles(rootPath string, wg *sync.WaitGroup) ([]worker, error) {
	var workers []worker
	dirEntries, err := os.ReadDir(rootPath)
	if err != nil {
		return nil, err
	}

	wg.Add(len(dirEntries))

	for i, dirEntry := range dirEntries {
		if !dirEntry.IsDir() {
			worker := createWorker(i, wg)
			workers = append(workers, worker)
			entryPath := filepath.Join(rootPath, dirEntry.Name())
			fileInfo, _ := dirEntry.Info()
			worker.in <- FileReceiveInfo{
				FilePath: entryPath,
				FileSize: fileInfo.Size(),
			}
		}
	}

	return workers, nil
}

const bufferSize = 65536

// MD5sum returns MD5 checksum of filename
func MD5sum(filename string) (string, error) {
	if info, err := os.Stat(filename); err != nil {
		return "", err
	} else if info.IsDir() {
		return "", nil
	}

	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	for buf, reader := make([]byte, bufferSize), bufio.NewReader(file); ; {
		n, err := reader.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			}
			return "", err
		}

		hash.Write(buf[:n])
	}

	checksum := fmt.Sprintf("%x", hash.Sum(nil))
	return checksum, nil
}

2022 年 6 月 28 日 11 点 28 分更新代码

Fix：程序遇到无权限或其他错误时会直接退出，改为输出错误，处理下一个文件 / 文件夹

package main

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

var (
	rootDir = "C:\\Users\\Administrator\\Downloads\\SM\\2016\\03\\19"
	// rootDir = "C:\\Users\\Administrator\\Downloads"
)

/*
Ref: [Go语言并发模型：以并行处理MD5为例 - SegmentFault 思否](https://segmentfault.com/a/1190000006670880)
*/
func main() {
	startTime := time.Now()

	rootPath := os.Args[1]

	fmt.Println("计算目录", rootPath)

	// 等待动画
	go spinner(250 * time.Millisecond)

	// 计算特定目录下所有文件的 md5值
	// 然后按照路径名顺序打印结果
	m, err := MD5All(rootPath)
	if err != nil {
		fmt.Printf("\r%v\n", err)
		return
	}
	var paths []string
	for path := range m {
		paths = append(paths, path)
	}
	sort.Strings(paths)

	result := make(map[string]string, len(paths))

	for _, path := range paths {
		// fmt.Printf("%x  %s\n", m[path], path)
		result[path] = fmt.Sprintf("%x", m[path])
	}

	fmt.Println("\r总耗时", time.Since(startTime), "总文件", len(result))

	marshal, _ := json.MarshalIndent(result, "", "    ")
	writeRelFile(rootPath, marshal)
}

func writeRelFile(rootPath string, marshal []byte) {
	writeFilePath := filepath.Join(rootPath, "marshal.json")
	err := ioutil.WriteFile(writeFilePath, marshal, 0755)
	if err != nil {
		dir, errwd := os.Getwd()
		if errwd != nil {
			fmt.Printf("\r获取程序所在目录出错：%v\n", errwd)
			return
		}
		if dir == rootPath {
			fmt.Println("\r当前文件所在目录无权限，请切换目录后执行")
			return
		}
		fmt.Printf("\r写入映射文件出错：%v，尝试写入到程序所在目录：%s\n", err, dir)
		writeRelFile(dir, marshal)
		return
	}
	fmt.Printf("\r映射文件已保存至【%s】目录。\n", writeFilePath)
}

func spinner(delay time.Duration) {
	for {
		for _, r := range `-\|/` {
			fmt.Printf("\r%c", r)
			time.Sleep(delay)
		}
	}
}

/*并发版 Start*/
type result struct {
	path string
	sum  [md5.Size]byte
	err  error
}

func sumFiles(done <-chan struct{}, root string) (<-chan result, <-chan error) {
	// 对于每一个普通文件，启动一个 gorotuine 计算文件 md5 值，
	// 然后 将结果发送到 c。
	// walk 的错误结果发送到 errc。
	c := make(chan result)
	errc := make(chan error, 1)
	go func() {
		var wg sync.WaitGroup
		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf("\rwalk has panic: %v\n", err)
				return filepath.SkipDir
			}
			if !info.Mode().IsRegular() {
				return nil
			}
			wg.Add(1)
			go func() {
				data, err := ioutil.ReadFile(path)
				select {
				case c <- result{path, md5.Sum(data), err}:
				case <-done:
				}
				wg.Done()
			}()
			// done channel 关闭时，终止 walk 函数
			select {
			case <-done:
				return errors.New("walk canceled")
			default:
				return nil
			}
		})
		// Walk 函数已经返回，所以 所有对 wg.Add 的调用都会结束
		// 启动一个 goroutine， 它会在所有发送都结束时，关闭 c。
		go func() {
			wg.Wait()
			close(c)
		}()
		// 这里不需要 select 语句，应为 errc 是缓冲管道
		errc <- err
	}()
	return c, errc
}

func MD5All(root string) (map[string][md5.Size]byte, error) {
	// MD5All 在函数返回时关闭 done channel
	// 在从 c 和 errc 接收数据前，也可能关闭
	done := make(chan struct{})
	defer close(done)

	c, errc := sumFiles(done, root)

	m := make(map[string][md5.Size]byte)
	for r := range c {
		if r.err != nil {
			return nil, fmt.Errorf("code in 138 has panic: %v", r.err)
		}
		m[r.path] = r.sum
	}
	if err := <-errc; err != nil {
		return nil, fmt.Errorf("code in 143 has panic: %v", err)
	}
	return m, nil
}

/*并发版 End*/

// ----------------------------------------- 分割线 -----------------------------------------

/*非并发版 Start*/
// MD5All 读取 root 目录下的所有文件，返回一个map
// 该 map 存储了 文件路径到文件内容 md5值的映射
// 如果 Walk 执行失败，或者 ioutil.ReadFile 读取失败，
// MD5All 都会返回错误
func MD5AllOld(root string) (map[string][md5.Size]byte, error) {
	m := make(map[string][md5.Size]byte)
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		m[path] = md5.Sum(data)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return m, nil
}

/*非并发版 End*/

2022 年 6 月 27 日 10 点 09 分更新代码

package main

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

var (
	// rootDir = "C:\\Users\\Administrator\\Downloads\\SM\\2016\\03\\19"
	rootDir = "C:\\Users\\Administrator\\Downloads"
)

/*
Ref: [Go语言并发模型：以并行处理MD5为例 - SegmentFault 思否](https://segmentfault.com/a/1190000006670880)
*/
func main() {
	startTime := time.Now()

	// 等待动画
	go spinner(100 * time.Millisecond)

	// 计算特定目录下所有文件的 md5值
	// 然后按照路径名顺序打印结果
	m, err := MD5All(rootDir)
	if err != nil {
		fmt.Println(err)
		return
	}
	var paths []string
	for path := range m {
		paths = append(paths, path)
	}
	sort.Strings(paths)

	result := make(map[string]string, len(paths))

	for _, path := range paths {
		// fmt.Printf("%x  %s\n", m[path], path)
		result[path] = fmt.Sprintf("%x", m[path])
	}

	fmt.Println("\r总耗时", time.Now().Sub(startTime))

	marshal, _ := json.MarshalIndent(result, "", "    ")
	writeFilePath := filepath.Join(rootDir, "marshal.json")
	err = ioutil.WriteFile(writeFilePath, marshal, 0755)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("\r映射文件已保存至【%s】目录。\n", writeFilePath)
}

func spinner(delay time.Duration) {
	for {
		for _, r := range `-\|/` {
			fmt.Printf("\r%c", r)
			time.Sleep(delay)
		}
	}
}

/*并发版 Start*/
type result struct {
	path string
	sum  [md5.Size]byte
	err  error
}

func sumFiles(done <-chan struct{}, root string) (<-chan result, <-chan error) {
	// 对于每一个普通文件，启动一个 gorotuine 计算文件 md5 值，
	// 然后 将结果发送到 c。
	// walk 的错误结果发送到 errc。
	c := make(chan result)
	errc := make(chan error, 1)
	go func() {
		var wg sync.WaitGroup
		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.Mode().IsRegular() {
				return nil
			}
			wg.Add(1)
			go func() {
				data, err := ioutil.ReadFile(path)
				select {
				case c <- result{path, md5.Sum(data), err}:
				case <-done:
				}
				wg.Done()
			}()
			// done channel 关闭时，终止 walk 函数
			select {
			case <-done:
				return errors.New("walk canceled")
			default:
				return nil
			}
		})
		// Walk 函数已经返回，所以 所有对 wg.Add 的调用都会结束
		// 启动一个 goroutine， 它会在所有发送都结束时，关闭 c。
		go func() {
			wg.Wait()
			close(c)
		}()
		// 这里不需要 select 语句，应为 errc 是缓冲管道
		errc <- err
	}()
	return c, errc
}

func MD5All(root string) (map[string][md5.Size]byte, error) {
	// MD5All 在函数返回时关闭 done channel
	// 在从 c 和 errc 接收数据前，也可能关闭
	done := make(chan struct{})
	defer close(done)

	c, errc := sumFiles(done, root)

	m := make(map[string][md5.Size]byte)
	for r := range c {
		if r.err != nil {
			return nil, r.err
		}
		m[r.path] = r.sum
	}
	if err := <-errc; err != nil {
		return nil, err
	}
	return m, nil
}

/*并发版 End*/

// ----------------------------------------- 分割线 -----------------------------------------

/*非并发版 Start*/
// MD5All 读取 root 目录下的所有文件，返回一个map
// 该 map 存储了 文件路径到文件内容 md5值的映射
// 如果 Walk 执行失败，或者 ioutil.ReadFile 读取失败，
// MD5All 都会返回错误
func MD5AllOld(root string) (map[string][md5.Size]byte, error) {
	m := make(map[string][md5.Size]byte)
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		m[path] = md5.Sum(data)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return m, nil
}

/*非并发版 End*/

2021 年 6 月 29 日 11 点 55 分更新代码

运行后将会在当前目录生成文件："当前目录名.txt" 的文件

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"io/ioutil"
	"os"
	"path/filepath"
)

// main 交叉编译命令：
// go tool dist list
/*
x86_x64的Linux: SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=amd64 go build hello.go
国产化arm: SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=arm64 go build hello.go
// go build -o hello.go
*/
func main() {
	startPath, _ := os.Getwd()
	fmt.Println("startPath:", startPath)

	paths, err := readDir(startPath)
	if nil != err {
		panic(err)
	}

	fmt.Printf("当前目录共计 \"%d\" 个文件\n", len(paths))

	// 获取文件 md5
	outputContent := ""
	for _, path := range paths {
		hash := md5.New()
		open, err := os.Open(path)
		if nil != err {
			panic(err)
		}

		_, _ = io.Copy(hash, open)

		sum := hash.Sum(nil)
		md5Str := hex.EncodeToString(sum)
		singleLine := fmt.Sprintf("%s -> %s\n", md5Str, path)
		outputContent += singleLine
	}

	// 准备写入文件
	fileName := fmt.Sprintf("%s.txt", filepath.Base(startPath))
	err = ioutil.WriteFile(fileName, []byte(outputContent), 0644)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
	} else {
		fmt.Printf("请查看当前目录下的 \"%s\" 文件\n", fileName)
	}

}

func readDir(pathname string) (paths []string, err error) {
	finalPath := pathname

	dir, err := ioutil.ReadDir(finalPath)
	if nil != err {
		return paths, err
	}

	for _, v := range dir {
		filename := v.Name()

		fullPath := fmt.Sprintf("%s%s%s", pathname, string(os.PathSeparator), filename)
		if v.IsDir() {
			err := filepath.WalkDir(fullPath, func(path string, d fs.DirEntry, err error) error {
				if nil != d && !d.IsDir() {
					paths = append(paths, path)
				}
				return err
			})
			if nil != err {
				fmt.Printf("%s\n", err.Error())
				return paths, err
			}
		} else {
			paths = append(paths, fullPath)
		}
	}

	return paths, nil
}

代码如下：

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"io/ioutil"
	"os"
	"path/filepath"
)

func main() {
	paths, err := readDir("C:\\Users\\biuaxia\\go\\src\\awesomeProject")
	if nil != err {
		panic(err)
	}

	fmt.Printf("%d\n", len(paths))

	for _, path := range paths {
		hash := md5.New()
		open, err := os.Open(path)
		if nil != err {
			panic(err)
		}

		_, _ = io.Copy(hash, open)

		sum := hash.Sum(nil)
		md5Str := hex.EncodeToString(sum)
		fmt.Printf("%s -> %s\n", md5Str, path)
	}
}

func readDir(pathname string) (paths []string, err error) {
	finalPath := pathname

	dir, err := ioutil.ReadDir(finalPath)
	if nil != err {
		return paths, err
	}

	for _, v := range dir {
		filename := v.Name()
		fullPath := fmt.Sprintf("%s\\%s", pathname, filename)
		if v.IsDir() {
			err := filepath.WalkDir(fullPath, func(path string, d fs.DirEntry, err error) error {
				if !d.IsDir() {
					paths = append(paths, path)
				}
				return err
			})
			if nil != err {
				panic(err)
				return paths, err
			}
		} else {
			paths = append(paths, fullPath)
		}
	}

	return paths, nil
}

目录结构为：

image.png

运行结果为：

image.png

TODO#

修复非 windows/amd64 操作系统无法使用的问题。

编译#

SET CGO_ENABLED=0
SET GOOS=linux
SET GOARCH=amd64
go build hello.go

查看当前 Go 支持编译的平台#

go tool dist list
載入中......
此文章數據所有權由區塊鏈加密技術和智能合約保障僅歸創作者所有。