8000 Hash tune files to avoid unnecessary re-fetch when configuring by vrn-sn · Pull Request #326 · luau-lang/lute · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Hash tune files to avoid unnecessary re-fetch when configuring #326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 5, 2025
Merged

Hash tune files to avoid unnecessary re-fetch when configuring #326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c861490
Add hashing for dependencies
vrn-sn Jun 5, 2025
23cccf0
Update luthier.py as well
vrn-sn Jun 5, 2025
795b069
Remove print
vrn-sn Jun 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion tools/luthier.luau
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,44 @@ local function fetchDependency(dependencyInfo: Tune)
end
end

local function getTuneFilesHash(): string
local externPath = projectRelative("extern")
local files = fs.listdir(externPath)

table.sort(files, function(a, b)
return a.name < b.name
end)

local toHash = ""

for _, file in files do
if not string.find(file.name, ".tune") then
continue
end

if file.type ~= "file" then
continue
end

toHash ..= file.name
toHash ..= fs.readfiletostring(joinPath(externPath, file.name))
end

local digest = crypto.digest(crypto.hash.blake2b256, toHash)
return hexify(digest)
end

local function areTuneFilesUpToDate(): boolean
local hashFile = projectRelative("extern", "generated", "hash.txt")

if safeFsType(hashFile) ~= "file" then
return false
end

local hash = fs.readfiletostring(hashFile)
return hash == getTuneFilesHash()
end

local function fetchDependencies(): number
local files = fs.listdir(projectRelative("extern"))

Expand All @@ -533,16 +571,32 @@ local function fetchDependencies(): number
end
end

local generatedPath = projectRelative("extern", "generated")
pcall(fs.mkdir, generatedPath)

local hash = fs.open(joinPath(generatedPath, "hash.txt"), "w+")
fs.write(hash, getTuneFilesHash())
fs.close(hash)

return 0
end

local function fetchDependenciesIfNeeded(): number
if areTuneFilesUpToDate() then
return 0
end

print("Fetching dependencies, tune files are out of date.")
return fetchDependencies()
end

local function configure()
local cmd = { "cmake" }
local arguments = getConfigureArguments()

table.move(arguments, 1, #arguments, 2, cmd)

check(fetchDependencies())
check(fetchDependenciesIfNeeded())
return call(cmd)
end

Expand Down
50 changes: 46 additions & 4 deletions tools/luthier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def getStdLibHash():
restoredPath = os.getcwd()
os.chdir(os.path.join(getSourceRoot(), 'lute/std/libs'))

hasher = blake2b()
hasher = blake2b(digest_size=32)
for dirpath, _, filenames in os.walk('.'):
for filename in sorted(filenames):
filepath = os.path.join(dirpath, filename)
Expand Down Expand Up @@ -273,7 +273,7 @@ def getCliCommandsHash():
restoredPath = os.getcwd()
os.chdir(os.path.join(getSourceRoot(), 'lute/cli/commands'))

hasher = blake2b()
hasher = blake2b(digest_size=32)
for dirpath, _, filenames in os.walk('.'):
for filename in sorted(filenames):
filepath = os.path.join(dirpath, filename)
Expand Down Expand Up @@ -494,23 +494,65 @@ def fetchDependency(dependencyInfo):
# if it doesn't exist, we'll do a shallow clone
return call(['git', 'clone', '--depth=1', '--branch', dependency['branch'], dependency['remote'], "extern/" + dependency['name']])

def getTuneFilesHash():
externPath = os.path.join(getSourceRoot(), "extern")
files = []
for entry in os.scandir(externPath):
if entry.is_file() and entry.name.endswith(".tune"):
files.append(entry.name)
files.sort()

hasher = blake2b(digest_size=32)
for fileName in files:
filePath = os.path.join(externPath, fileName)
with open(filePath, "r") as f:
hasher.update(fileName.encode('utf-8'))
hasher.update(f.read().encode('utf-8'))

return hasher.hexdigest()

def areTuneFilesUpToDate():
hashFile = os.path.join(getSourceRoot(), "extern", "generated", "hash.txt")
if not os.path.isfile(hashFile):
return False

with open(hashFile, 'r') as f:
lines = f.readlines()
assert(len(lines) == 1)
actual = lines[0]
expected = getTuneFilesHash()
return actual == expected

def fetchDependencies(args):
for _, _, files in os.walk('extern'):
for file in files:
if file.endswith('.tune'):
dependencyInfo = readTuneFile(os.path.join('extern', file))
check(fetchDependency(dependencyInfo))

generatedPath = os.path.join(getSourceRoot(), "extern", "generated")
os.makedirs(generatedPath, exist_ok=True)

hashFilePath = os.path.join(generatedPath, "hash.txt")
with open(hashFilePath, "w") as hash:
hash.write(getTuneFilesHash())

return 0

def fetchDependenciesIfNeeded(args):
if areTuneFilesUpToDate():
return 0

print("Fetching dependencies, tune files are out of date.")
return fetchDependencies(args)

def configure(args):
"""Runs any necessary configuration steps to generate build files"""
cmd = [
"cmake",
] + getConfigureArguments(args)

# fetchDependencies is too slow.
# check(fetchDependencies(args))
check(fetchDependenciesIfNeeded(args))
return call(cmd)

def check(exitCode):
Expand Down
0