Skip to content

Commit c067624

Browse files
XrekiXreki
committed
Merge branch 'develop' into opt_saved_results
2 parents 1196549 + de2ba85 commit c067624

File tree

78 files changed

+10568
-49
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+10568
-49
lines changed

graph_net/paddle/naive_graph_decomposer.py renamed to graph_net/paddle/graph_decomposer.py

File renamed without changes.

‎graph_net/subgraph_decompose_and_evaluation_step.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def run_decomposer_for_single_model(
191191
"decorator_path": f"{graphnet_root}/graph_net/{framework}/extractor.py",
192192
"decorator_config": {
193193
"name": model_name,
194-
"custom_extractor_path": f"{graphnet_root}/graph_net/{framework}/naive_graph_decomposer.py",
194+
"custom_extractor_path": f"{graphnet_root}/graph_net/{framework}/graph_decomposer.py",
195195
"custom_extractor_config": {
196196
"output_dir": output_dir,
197197
"split_positions": split_positions,

‎graph_net/tensor_meta.py‎

100644100755
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ def serialize_to_py_str(self) -> str:
5858
lines = [
5959
(f"class {self.record_class_name}:"),
6060
(f'\tname = "{self.name}"'),
61+
*(
62+
[f'\toriginal_name = "{self.original_name}"']
63+
if self.original_name is not None
64+
else []
65+
),
6166
(f"\tshape = {self.shape}"),
6267
(f'\tdtype = "{self.dtype}"'),
6368
(f'\tdevice = "{self.device}"'),

‎graph_net/test/chain_naive_graph_decomposer_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ decorator_config_json_str=$(cat <<EOF
1111
"decorator_path": "$GRAPH_NET_ROOT/torch/extractor.py",
1212
"decorator_config": {
1313
"name": "$MODEL_NAME",
14-
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/naive_graph_decomposer.py",
14+
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/graph_decomposer.py",
1515
"custom_extractor_config": {
1616
"output_dir": "/tmp/chain_naive_decompose_workspace",
1717
"split_positions": [8, 16, 32],

‎graph_net/test/decomposer_validator_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ extractor_config_json_str=$(cat <<EOF
1616
"decorator_path": "$GRAPH_NET_ROOT/torch/extractor.py",
1717
"decorator_config": {
1818
"name": "$MODEL_NAME",
19-
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/naive_graph_decomposer.py",
19+
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/graph_decomposer.py",
2020
"custom_extractor_config": {
2121
"output_dir": "$OUTPUT_DIR/${MODEL_NAME}_decomposed",
2222
"split_positions": [8, 16, 32],

‎graph_net/test/graph_variable_rename_test.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(
4+
os.path.dirname(graph_net.__file__))")
5+
6+
# input model path
7+
MODEL_NAME=resnet18
8+
MODEL_PATH_IN_SAMPLES=/timm/$MODEL_NAME
9+
config_json_str=$(cat <<EOF
10+
{
11+
"handler_path": "$GRAPH_NET_ROOT/torch/graph_variable_renamer.py",
12+
"handler_class_name": "GraphVariableRenamer",
13+
"handler_config": {
14+
"model_path_prefix": "$GRAPH_NET_ROOT/../",
15+
"data_input_predicator_filepath": "$GRAPH_NET_ROOT/torch/constraint_util.py",
16+
"data_input_predicator_class_name": "NaiveDataInputPredicator",
17+
"model_runnable_predicator_filepath": "$GRAPH_NET_ROOT/torch/constraint_util.py",
18+
"model_runnable_predicator_class_name": "ModelRunnablePredicator",
19+
"output_dir": "/tmp/graph_variable_rename_workspace"
20+
}
21+
}
22+
EOF
23+
)
24+
CONFIG=$(echo $config_json_str | base64 -w 0)
25+
26+
python3 -m graph_net.model_path_handler --model-path samples/$MODEL_PATH_IN_SAMPLES --handler-config=$CONFIG
27+
# python3 -m graph_net.model_path_handler --model-path-list $GRAPH_NET_ROOT/config/decomposition_error_tmp_torch_samples_list.txt --handler-config=$CONFIG

‎graph_net/test/naive_decomposer_and_post_extract_process_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ decorator_config_json_str=$(cat <<EOF
1212
"decorator_path": "$GRAPH_NET_ROOT/torch/extractor.py",
1313
"decorator_config": {
1414
"name": "$MODEL_NAME",
15-
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/naive_graph_decomposer.py",
15+
"custom_extractor_path": "$GRAPH_NET_ROOT/torch/graph_decomposer.py",
1616
"custom_extractor_config": {
1717
"output_dir": "/tmp/naive_decompose_workspace",
1818
"split_positions": [8, 16, 32],

‎graph_net/test/naive_graph_decomposer_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ MODEL_NAME=resnet18
88
MODEL_PATH_IN_SAMPLES=/timm/$MODEL_NAME
99
config_json_str=$(cat <<EOF
1010
{
11-
"handler_path": "$GRAPH_NET_ROOT/torch/naive_graph_decomposer.py",
11+
"handler_path": "$GRAPH_NET_ROOT/torch/graph_decomposer.py",
1212
"handler_class_name": "NaiveDecomposerExtractor",
1313
"handler_config": {
1414
"model_path_prefix": "$GRAPH_NET_ROOT/../",

‎graph_net/test/typical_sequence_decomposer_test.sh‎

Lines changed: 36 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,57 @@
11
#!/bin/bash
22

33
GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(os.path.dirname(os.path.dirname(graph_net.__file__)))")
4-
DECOMPOSE_PATH=$GRAPH_NET_ROOT/decompose_workspace
4+
DECOMPOSE_PATH=/tmp/decompose_workspace
55

66
mkdir -p "$DECOMPOSE_PATH"
77

8-
temp_model_list=$(mktemp)
9-
cat "$GRAPH_NET_ROOT/graph_net/config/torch_samples_list.txt" > "$temp_model_list"
8+
model_list="$GRAPH_NET_ROOT/graph_net/config/small100_torch_samples_list.txt"
109

1110
python3 -m graph_net.torch.typical_sequence_split_points \
12-
--model-list "$temp_model_list" \
11+
--model-list "$model_list" \
1312
--device "cuda" \
1413
--window-size 10 \
14+
--fold-policy default \
15+
--fold-times 10 \
1516
--output-json "$DECOMPOSE_PATH/split_results.json"
1617

17-
while IFS= read -r MODEL_PATH_IN_SAMPLES; do
18-
if [[ -n "$MODEL_PATH_IN_SAMPLES" ]]; then
19-
MODEL_FULL_PATH="$GRAPH_NET_ROOT/$MODEL_PATH_IN_SAMPLES"
20-
MODEL_NAME=$(basename "$MODEL_PATH_IN_SAMPLES")
21-
22-
echo "== Decomposing $MODEL_PATH_IN_SAMPLES. =="
23-
24-
decomposer_config_json_str=$(cat <<EOF
18+
decompose_config_json_str=$(cat <<EOF
2519
{
26-
"split_results_path": "$DECOMPOSE_PATH/split_results.json",
27-
"workspace_path": "$DECOMPOSE_PATH",
28-
"chain_style": true,
29-
"target_model_name": "$MODEL_NAME"
20+
"handler_path": "$GRAPH_NET_ROOT/graph_net/torch/graph_decomposer.py",
21+
"handler_class_name": "RangeDecomposerExtractor",
22+
"handler_config": {
23+
"model_path_prefix": "$GRAPH_NET_ROOT",
24+
"output_dir": "$DECOMPOSE_PATH",
25+
"split_results_path": "$DECOMPOSE_PATH/split_results.json",
26+
"group_head_and_tail": true,
27+
"chain_style": true
28+
}
3029
}
3130
EOF
32-
)
33-
DECOMPOSER_CONFIG=$(echo $decomposer_config_json_str | base64 -w 0)
34-
35-
python3 -m graph_net.torch.test_compiler \
36-
--model-path "$MODEL_FULL_PATH" \
37-
--compiler range_decomposer \
38-
--device cuda \
39-
--config="$DECOMPOSER_CONFIG"
40-
41-
cp -r "$MODEL_FULL_PATH" "$DECOMPOSE_PATH/"
42-
43-
echo "== Validating $MODEL_PATH_IN_SAMPLES. =="
31+
)
32+
DECOMPOSE_CONFIG=$(echo $decompose_config_json_str | base64 -w 0)
4433

45-
python3 -m graph_net.torch.test_compiler \
46-
--model-path "$DECOMPOSE_PATH/$MODEL_NAME" \
47-
--compiler range_decomposer_validator \
48-
--device cuda > "$DECOMPOSE_PATH/${MODEL_NAME}_validation.log" 2>&1
34+
python3 -m graph_net.model_path_handler \
35+
--model-path-list $model_list \
36+
--handler-config=$DECOMPOSE_CONFIG \
37+
--use-subprocess
4938

50-
echo "== Finished processing $MODEL_PATH_IN_SAMPLES. =="
51-
fi
52-
done < $temp_model_list
53-
54-
rm -f "$temp_model_list"
39+
test_compiler_config_json_str=$(cat <<EOF
40+
{
41+
"decomposed_root": "$DECOMPOSE_PATH"
42+
}
43+
EOF
44+
)
45+
TEST_COMPILER_CONFIG=$(echo $test_compiler_config_json_str | base64 -w 0)
5546

56-
cat $DECOMPOSE_PATH/*_validation.log >> $DECOMPOSE_PATH/combined.log
47+
python3 -m graph_net.torch.test_compiler \
48+
--allow-list $model_list \
49+
--compiler range_decomposer_validator \
50+
--device cuda \
51+
--config $TEST_COMPILER_CONFIG \
52+
--model-path-prefix $GRAPH_NET_ROOT \
53+
> "$DECOMPOSE_PATH/validation.log" 2>&1
5754

5855
python3 -m graph_net.plot_ESt \
59-
--benchmark-path "$DECOMPOSE_PATH/combined.log" \
56+
--benchmark-path "$DECOMPOSE_PATH/validation.log" \
6057
--output-dir "$DECOMPOSE_PATH"

‎graph_net/torch/backend/range_decomposer_backend.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def __call__(self, model: torch.nn.Module) -> torch.nn.Module:
4444
"decorator_config": {
4545
"name": model_name,
4646
"custom_extractor_path": str(
47-
self.graph_net_root / "torch/naive_graph_decomposer.py"
47+
self.graph_net_root / "torch/graph_decomposer.py"
4848
),
4949
"custom_extractor_config": {
5050
"output_dir": str(model_output_dir),

0 commit comments

Comments
 (0)