Add reference test for AutoMerge comment (and all its permutations)

Project.toml

@@ -34,6 +34,7 @@ LicenseCheck = "0.2"
 Pkg = "1"
 Printf = "<0.0.1, 1"
 Random = "<0.0.1, 1"
+ReferenceTests = "0.9, 0.10"
 RegistryTools = "2.2"
 SHA = "<0.0.1, 0.7, 1"
 SimpleMock = "1"
@@ -52,8 +53,9 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 SimpleMock = "a896ed2c-15a5-4479-b61d-a0e88e2a1d25"
+ReferenceTests = "324d217c-45ce-50fc-942e-d289b448e8cf"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 
 [targets]
-test = ["Dates", "GitHub", "JSON", "Pkg", "Printf", "SimpleMock", "Test", "TimeZones"]
+test = ["Dates", "GitHub", "JSON", "Pkg", "Printf", "SimpleMock", "ReferenceTests", "Test", "TimeZones"]

test/automerge-unit.jl

@@ -45,7 +45,45 @@ function pkgdir_from_depot(depot_path::String, pkg::String)
     return only_pkdir
 end
 
+# Here we reference test all the permutations of the AutoMerge comments.
+# This allows us to see the diffs in PRs that change the AutoMerge comment.
+function comment_reference_test()
+    for pass in (true, false),
+        (type_name,type) in (("new_version", AutoMerge.NewVersion()), ("new_package", AutoMerge.NewPackage())),
+        suggest_onepointzero in (true, false),
+        # some code depends on above or below v"1"
+        version in (v"0.1", v"1")
+
+        if pass
+            for is_jll in (true, false)
+                name = string("comment", "_pass_", pass, "_type_", type_name,
+                "_suggest_onepointzero_", suggest_onepointzero,
+                "_version_", version, "_is_jll_", is_jll)
+                @test_reference "reference_comments/$name.md" AutoMerge.comment_text_pass(type, suggest_onepointzero, version, is_jll)
+            end
+        else
+            for point_to_slack in (true, false)
+                name = string("comment", "_pass_", pass, "_type_", type_name,
+                "_suggest_onepointzero_", suggest_onepointzero,
+                "_version_", version, "_point_to_slack_", point_to_slack)
+                reasons = ["Example guideline failed. Please fix it."]
+                fail_text = AutoMerge.comment_text_fail(type, reasons, suggest_onepointzero, version; point_to_slack=point_to_slack)
+
+                @test_reference "reference_comments/$name.md" fail_text
+
+                # `point_to_slack=false` should yield no references to Slack in the text
+                if !point_to_slack
+                    @test !occursin("slack", fail_text)
+                end
+            end
+        end
+    end
+end
+
 @testset "Utilities" begin
+    @testset "comment_reference_test" begin
+        comment_reference_test()
+    end
     @testset "`AutoMerge.parse_registry_pkg_info`" begin
         registry_path = joinpath(DEPOT_PATH[1], "registries", "General")
         result = AutoMerge.parse_registry_pkg_info(registry_path, "RegistryCI", "1.0.0")

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_false_version_0.1.0_point_to_slack_false.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_false_version_0.1.0_point_to_slack_true.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged. Then, send a message to the `#pkg-registration` channel in the [Julia Slack](https://julialang.org/slack/) to ask for help. Include a link to this pull request.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_false_version_1.0.0_point_to_slack_false.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_false_version_1.0.0_point_to_slack_true.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged. Then, send a message to the `#pkg-registration` channel in the [Julia Slack](https://julialang.org/slack/) to ask for help. Include a link to this pull request.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_true_version_0.1.0_point_to_slack_false.md

@@ -0,0 +1,24 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+
+---
+On a separate note, I see that you are registering a release with a version number of the form `v0.X.Y`.
+
+Does your package have a stable public API? If so, then it's time for you to register version `v1.0.0` of your package. (This is not a requirement. It's just a recommendation.)
+
+If your package does not yet have a stable public API, then of course you are not yet ready to release version `v1.0.0`.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_true_version_0.1.0_point_to_slack_true.md

@@ -0,0 +1,24 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged. Then, send a message to the `#pkg-registration` channel in the [Julia Slack](https://julialang.org/slack/) to ask for help. Include a link to this pull request.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+
+---
+On a separate note, I see that you are registering a release with a version number of the form `v0.X.Y`.
+
+Does your package have a stable public API? If so, then it's time for you to register version `v1.0.0` of your package. (This is not a requirement. It's just a recommendation.)
+
+If your package does not yet have a stable public API, then of course you are not yet ready to release version `v1.0.0`.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_true_version_1.0.0_point_to_slack_false.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_package_suggest_onepointzero_true_version_1.0.0_point_to_slack_true.md

@@ -0,0 +1,17 @@
+Your `new package` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged. Then, send a message to the `#pkg-registration` channel in the [Julia Slack](https://julialang.org/slack/) to ask for help. Include a link to this pull request.
+
+Since you are registering a new package, please make sure that you have also read the package naming guidelines: https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-guidelines
+
+
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->

test/reference_comments/comment_pass_false_type_new_version_suggest_onepointzero_false_version_0.1.0_point_to_slack_false.md

@@ -0,0 +1,13 @@
+Your `new version` pull request does not meet the guidelines for auto-merging. Please make sure that you have read the [General registry README](https://github.com/JuliaRegistries/General/blob/master/README.md) and the [AutoMerge guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). The following guidelines were not met:
+
+- Example guideline failed. Please fix it.
+
+Note that the guidelines are only required for the pull request to be merged automatically. However, it is **strongly recommended** to follow them, since otherwise the pull request needs to be manually reviewed and merged by a human.
+
+After you have fixed the AutoMerge issues, simply retrigger Registrator, which will automatically update this pull request. You do not need to change the version number in your `Project.toml` file (unless of course the AutoMerge issue is that you skipped a version number, in which case you should change the version number).
+
+If you do not want to fix the AutoMerge issues, please post a comment explaining why you would like this pull request to be manually merged.
+
+---
+If you want to prevent this pull request from being auto-merged, simply leave a comment. If you want to post a comment without blocking auto-merging, you must include the text `[noblock]` in your comment. You can edit blocking comments, adding `[noblock]` to them in order to unblock auto-merging.
+<!-- [noblock] -->