Closed
Description
🐛 Describe the bug
Testing ToT triton before release/2.8 to assess the issues.
This test is confirmed to be amd specific.
Affected tests: 2
error message:
{'message': 'torch._inductor.exc.InductorError: SubprocException: An exception occurred in a subprocess:\n\nTraceback (most recent call last):\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 327, in do_job\n result = job()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/compile_tasks.py", line 57, in _worker_compile_triton\n kernel.precompile(warm_cache_only=True)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 313, in precompile\n self._precompile_worker()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 344, in _precompile_worker\n compile_results.append(self._precompile_config(c))\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 627, in _precompile_config\n binary = triton.compile(*compile_args, **compile_kwargs)\n File "/tmp/triton/python/triton/compiler/compiler.py", line 349, in compile\n next_module = compile_ir(module, metadata)\n File "/tmp/triton/python/triton/backends/amd/compiler.py", line 427, in <lambda>\n stages["hsaco"] = lambda src, metadata: self.make_hsaco(src, metadata, options)\n File "/tmp/triton/python/triton/backends/amd/compiler.py", line 417, in make_hsaco\n subprocess.check_call([rocm_path, \'-flavor\', \'gnu\', \'-shared\', tmp_in.name, \'-o\', tmp_out.name])\n File "/opt/conda/envs/py_3.10/lib/python3.10/subprocess.py", line 369, in check_call\n raise CalledProcessError(retcode, cmd)\nsubprocess.CalledProcessError: Command \'[PosixPath(\'/opt/rocm/llvm/bin/ld.lld\'), \'-flavor\', \'gnu\', \'-shared\', \'/tmp/tmpahmfo1wk\', \'-o\', \'/tmp/tmpr3u866vu\']\' returned non-zero exit status 1.\n\n\nSet TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you\'re reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"\n\n\nTo execute this test, run the following from the base repo dir:\n python test/inductor/test_custom_lowering.py TestCustomLowering.test_multi_inp_asm\n\nThis message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0', 'text': 'Traceback (most recent call last):\n File "/tmp/pytorch/test/inductor/test_custom_lowering.py", line 219, in test_multi_inp_asm\n out2 = fn_opt(a, b)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 676, in _fn\n raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 880, in _compile_fx_inner\n raise InductorError(e, currentframe()).with_traceback(\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 864, in _compile_fx_inner\n mb_compiled_graph = fx_codegen_and_compile(\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1487, in fx_codegen_and_compile\n return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1374, in codegen_and_compile\n compiled_module = graph.compile_to_module()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2279, in compile_to_module\n return self._compile_to_module()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2289, in _compile_to_module\n mod = self._compile_to_module_lines(wrapper_code)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2353, in _compile_to_module_lines\n mod = PyCodeCache.load_by_key_path(\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 3036, in load_by_key_path\n mod = _reload_python_module(key, path, set_sys_modules=in_toplevel)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/compile_tasks.py", line 31, in _reload_python_module\n exec(code, mod.__dict__, mod.__dict__)\n File "/tmp/tmpqsssrgdj/jp/cjpoknzjrtwnpqvirdss24d4p7fuubawfuxns5nkntvs3vqd2tl7.py", line 73, in <module>\n async_compile.wait(globals())\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 481, in wait\n self._wait_futures(scope)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 501, in _wait_futures\n kernel = result.result()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 3561, in result\n return self.result_fn()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 362, in get_result\n kernel, elapsed_us = task.result()\n File "/opt/conda/envs/py_3.10/lib/python3.10/concurrent/futures/_base.py", line 458, in result\n return self.__get_result()\n File "/opt/conda/envs/py_3.10/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result\n raise self._exception\ntorch._inductor.exc.InductorError: SubprocException: An exception occurred in a subprocess:\n\nTraceback (most recent call last):\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 327, in do_job\n result = job()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/compile_tasks.py", line 57, in _worker_compile_triton\n kernel.precompile(warm_cache_only=True)\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 313, in precompile\n self._precompile_worker()\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 344, in _precompile_worker\n compile_results.append(self._precompile_config(c))\n File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 627, in _precompile_config\n binary = triton.compile(*compile_args, **compile_kwargs)\n File "/tmp/triton/python/triton/compiler/compiler.py", line 349, in compile\n next_module = compile_ir(module, metadata)\n File "/tmp/triton/python/triton/backends/amd/compiler.py", line 427, in <lambda>\n stages["hsaco"] = lambda src, metadata: self.make_hsaco(src, metadata, options)\n File "/tmp/triton/python/triton/backends/amd/compiler.py", line 417, in make_hsaco\n subprocess.check_call([rocm_path, \'-flavor\', \'gnu\', \'-shared\', tmp_in.name, \'-o\', tmp_out.name])\n File "/opt/conda/envs/py_3.10/lib/python3.10/subprocess.py", line 369, in check_call\n raise CalledProcessError(retcode, cmd)\nsubprocess.CalledProcessError: Command \'[PosixPath(\'/opt/rocm/llvm/bin/ld.lld\'), \'-flavor\', \'gnu\', \'-shared\', \'/tmp/tmpahmfo1wk\', \'-o\', \'/tmp/tmpr3u866vu\']\' returned non-zero exit status 1.\n\n\nSet TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you\'re reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"\n\n\nTo execute this test, run the following from the base repo dir:\n python test/inductor/test_custom_lowering.py TestCustomLowering.test_multi_inp_asm\n\nThis message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0'}
Versions
upstream pytorch + triton commit: triton-lang/triton@2ec711b
cc @jeffdaily @sunway513 @jithunnair-amd @pruthvistony @ROCmSupport @dllehr-amd @jataylo @hongxiayang @naromero77amd @chauhang @penguinwu @bertmaher @int3 @davidberard98 @nmacchioni @chenyang78 @embg @peterbell10 @aakhundov
Metadata
Metadata
Assignees
Labels
Type
Projects
Status
Done