Fixed comment parsing, added generated docs

This commit is contained in:
Mike Farah
2025-12-08 11:42:07 +11:00
parent 554bf5a2f2
commit 77eccfd3db
8 changed files with 382 additions and 48 deletions

View File

@@ -201,7 +201,7 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
leadingUsed = true
}
if headComment != "" {
valNode.HeadComment = headComment
keyNode.HeadComment = headComment
}
if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" {
valNode.LineComment = lineComment
@@ -229,7 +229,7 @@ func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode {
// Attach comments if any
attrRange := attrWithName.Attr.Range()
if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" {
val.HeadComment = headComment
key.HeadComment = headComment
}
if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" {
val.LineComment = lineComment
@@ -428,11 +428,11 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
if start >= 0 && end >= start && end <= len(src) {
text := strings.TrimSpace(string(src[start:end]))
node := createStringScalarNode(text)
node.Style = LiteralStyle
node.Style = 0
return node
}
node := createStringScalarNode(e.Name)
node.Style = LiteralStyle
node.Style = 0
return node
default:
// try to evaluate the expression (handles unary, binary ops, etc.)
@@ -447,9 +447,9 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
end := r.End.Byte
if start >= 0 && end >= start && end <= len(src) {
text := string(src[start:end])
// Mark as raw expression so encoder can emit without quoting
// Mark as unquoted expression so encoder emits without quoting
node := createStringScalarNode(text)
node.Style = LiteralStyle
node.Style = 0
return node
}
return createStringScalarNode(fmt.Sprintf("%v", expr))

177
pkg/yqlib/doc/usage/hcl.md Normal file
View File

@@ -0,0 +1,177 @@
# HCL
Encode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl).
HCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support:
- Blocks and attributes
- String interpolation and expressions (preserved without quotes)
- Comments (leading, head, and line comments)
- Nested structures (maps and lists)
- Syntax colorization when enabled
## Parse HCL
Given a sample.hcl file of:
```hcl
io_mode = "async"
```
then
```bash
yq -oy sample.hcl
```
will output
```yaml
io_mode: "async"
```
## Roundtrip: Sample Doc
Given a sample.hcl file of:
```hcl
service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
```
then
```bash
yq sample.hcl
```
will output
```hcl
service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
```
## Roundtrip: With an update
Given a sample.hcl file of:
```hcl
service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
```
then
```bash
yq '.service.cat.process.main.command += "meow"' sample.hcl
```
will output
```hcl
service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server", "meow"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
```
## Parse HCL: Sample Doc
Given a sample.hcl file of:
```hcl
service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
```
then
```bash
yq -oy sample.hcl
```
will output
```yaml
service:
cat:
process:
main:
command:
- "/usr/local/bin/awesome-app"
- "server"
management:
command:
- "/usr/local/bin/awesome-app"
- "management"
```
## Parse HCL: with comments
Given a sample.hcl file of:
```hcl
# Configuration
port = 8080 # server port
```
then
```bash
yq -oy sample.hcl
```
will output
```yaml
# Configuration
port: 8080 # server port
```
## Roundtrip: with comments
Given a sample.hcl file of:
```hcl
# Configuration
port = 8080
```
then
```bash
yq sample.hcl
```
will output
```hcl
# Configuration
port = 8080
```
## Roundtrip: With templates, functions and arithmetic
Given a sample.hcl file of:
```hcl
# Arithmetic with literals and application-provided variables
sum = 1 + addend
# String interpolation and templates
message = "Hello, ${name}!"
# Application-provided functions
shouty_message = upper(message)
```
then
```bash
yq sample.hcl
```
will output
```hcl
# Arithmetic with literals and application-provided variables
sum = 1 + addend
# String interpolation and templates
message = "Hello, ${name}!"
# Application-provided functions
shouty_message = upper(message)
```

View File

@@ -0,0 +1,11 @@
# HCL
Encode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl).
HCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support:
- Blocks and attributes
- String interpolation and expressions (preserved without quotes)
- Comments (leading, head, and line comments)
- Nested structures (maps and lists)
- Syntax colorization when enabled

View File

@@ -53,7 +53,7 @@ Given a sample.xml file of:
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -100,7 +100,7 @@ Given a sample.xml file of:
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -157,7 +157,7 @@ Given a sample.xml file of:
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -177,7 +177,7 @@ Given a sample.xml file of:
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -196,7 +196,7 @@ Given a sample.xml file of:
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -225,7 +225,7 @@ Given a sample.xml file of:
```
then
```bash
yq '.' sample.xml
yq sample.xml
```
will output
```xml
@@ -256,7 +256,7 @@ Given a sample.xml file of:
```
then
```bash
yq --xml-skip-directives '.' sample.xml
yq --xml-skip-directives sample.xml
```
will output
```xml
@@ -292,7 +292,7 @@ for x --></x>
```
then
```bash
yq -oy '.' sample.xml
yq -oy sample.xml
```
will output
```yaml
@@ -327,7 +327,7 @@ Given a sample.xml file of:
```
then
```bash
yq --xml-keep-namespace=false '.' sample.xml
yq --xml-keep-namespace=false sample.xml
```
will output
```xml
@@ -361,7 +361,7 @@ Given a sample.xml file of:
```
then
```bash
yq --xml-raw-token=false '.' sample.xml
yq --xml-raw-token=false sample.xml
```
will output
```xml
@@ -542,7 +542,7 @@ for x --></x>
```
then
```bash
yq '.' sample.xml
yq sample.xml
```
will output
```xml
@@ -575,7 +575,7 @@ Given a sample.xml file of:
```
then
```bash
yq '.' sample.xml
yq sample.xml
```
will output
```xml

View File

@@ -80,7 +80,7 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
return
}
// For mapping nodes, collect comments from values
// For mapping nodes, collect comments from keys and values
if node.Kind == MappingNode {
// Collect root-level head comment if at root (prefix is empty)
if prefix == "" && node.HeadComment != "" {
@@ -98,10 +98,11 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
path = prefix + "." + key
}
// Store comments for this value
if valueNode.HeadComment != "" {
commentMap[path+".head"] = valueNode.HeadComment
// Store comments from the key (head comments appear before the attribute)
if keyNode.HeadComment != "" {
commentMap[path+".head"] = keyNode.HeadComment
}
// Store comments from the value (line comments appear after the value)
if valueNode.LineComment != "" {
commentMap[path+".line"] = valueNode.LineComment
}
@@ -344,6 +345,15 @@ func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) {
// encodeAttribute encodes a value as an HCL attribute
func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode *CandidateNode) error {
if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" {
// Handle unquoted expressions (as-is, without quotes)
if valueNode.Style == 0 {
tokens, err := tokensForRawHCLExpr(valueNode.Value)
if err != nil {
return err
}
body.SetAttributeRaw(key, tokens)
return nil
}
if valueNode.Style&LiteralStyle != 0 {
tokens, err := tokensForRawHCLExpr(valueNode.Value)
if err != nil {

View File

@@ -1,6 +1,8 @@
package yqlib
import (
"bufio"
"fmt"
"testing"
"github.com/mikefarah/yq/v4/test"
@@ -32,6 +34,16 @@ var multipleBlockLabelKeysExpected = `service "cat" {
}
`
var multipleBlockLabelKeysExpectedUpdate = `service "cat" {
process "main" {
command = ["/usr/local/bin/awesome-app", "server", "meow"]
}
process "management" {
command = ["/usr/local/bin/awesome-app", "management"]
}
}
`
var multipleBlockLabelKeysExpectedYaml = `service:
cat:
process:
@@ -45,7 +57,7 @@ var multipleBlockLabelKeysExpectedYaml = `service:
- "management"
`
var roundtripSample = `# Arithmetic with literals and application-provided variables
var simpleSample = `# Arithmetic with literals and application-provided variables
sum = 1 + addend
# String interpolation and templates
@@ -54,7 +66,7 @@ message = "Hello, ${name}!"
# Application-provided functions
shouty_message = upper(message)`
var roundtripSampleExpected = `# Arithmetic with literals and application-provided variables
var simpleSampleExpected = `# Arithmetic with literals and application-provided variables
sum = 1 + addend
# String interpolation and templates
message = "Hello, ${name}!"
@@ -62,217 +74,275 @@ message = "Hello, ${name}!"
shouty_message = upper(message)
`
var simpleSampleExpectedYaml = `# Arithmetic with literals and application-provided variables
sum: 1 + addend
# String interpolation and templates
message: "Hello, ${name}!"
# Application-provided functions
shouty_message: upper(message)
`
var hclFormatScenarios = []formatScenario{
{
description: "Simple decode",
description: "Parse HCL",
input: `io_mode = "async"`,
expected: "io_mode: \"async\"\n",
scenarioType: "decode",
},
{
description: "Simple decode, no quotes",
skipDoc: true,
input: `io_mode = async`,
expected: "io_mode: async\n",
scenarioType: "decode",
},
{
description: "Simple roundtrip, no quotes",
skipDoc: true,
input: `io_mode = async`,
expected: "io_mode = async\n",
scenarioType: "roundtrip",
},
{
description: "Nested decode",
skipDoc: true,
input: nestedExample,
expected: nestedExampleYaml,
scenarioType: "decode",
},
{
description: "Template decode",
skipDoc: true,
input: `message = "Hello, ${name}!"`,
expected: "message: \"Hello, ${name}!\"\n",
scenarioType: "decode",
},
{
description: "Template roundtrip",
description: "Roundtrip: with template",
skipDoc: true,
input: `message = "Hello, ${name}!"`,
expected: "message = \"Hello, ${name}!\"\n",
scenarioType: "roundtrip",
},
{
description: "Function roundtrip",
description: "Roundtrip: with function",
skipDoc: true,
input: `shouty_message = upper(message)`,
expected: "shouty_message = upper(message)\n",
scenarioType: "roundtrip",
},
{
description: "Arithmetic roundtrip",
description: "Roundtrip: with arithmetic",
skipDoc: true,
input: `sum = 1 + addend`,
expected: "sum = 1 + addend\n",
scenarioType: "roundtrip",
},
{
description: "Arithmetic decode",
skipDoc: true,
input: `sum = 1 + addend`,
expected: "sum: 1 + addend\n",
scenarioType: "decode",
},
{
description: "number attribute",
skipDoc: true,
input: `port = 8080`,
expected: "port: 8080\n",
scenarioType: "decode",
},
{
description: "float attribute",
skipDoc: true,
input: `pi = 3.14`,
expected: "pi: 3.14\n",
scenarioType: "decode",
},
{
description: "boolean attribute",
skipDoc: true,
input: `enabled = true`,
expected: "enabled: true\n",
scenarioType: "decode",
},
{
description: "list of strings",
input: `tags = ["a", "b"]`,
expected: "tags:\n - \"a\"\n - \"b\"\n",
scenarioType: "decode",
},
{
description: "object/map attribute",
skipDoc: true,
input: `obj = { a = 1, b = "two" }`,
expected: "obj: {a: 1, b: \"two\"}\n",
scenarioType: "decode",
},
{
description: "nested block",
skipDoc: true,
input: `server { port = 8080 }`,
expected: "server:\n port: 8080\n",
scenarioType: "decode",
},
{
description: "multiple attributes",
skipDoc: true,
input: "name = \"app\"\nversion = 1\nenabled = true",
expected: "name: \"app\"\nversion: 1\nenabled: true\n",
scenarioType: "decode",
},
{
description: "binary expression",
skipDoc: true,
input: `count = 0 - 42`,
expected: "count: -42\n",
scenarioType: "decode",
},
{
description: "negative number",
skipDoc: true,
input: `count = -42`,
expected: "count: -42\n",
scenarioType: "decode",
},
{
description: "scientific notation",
skipDoc: true,
input: `value = 1e-3`,
expected: "value: 0.001\n",
scenarioType: "decode",
},
{
description: "nested object",
skipDoc: true,
input: `config = { db = { host = "localhost", port = 5432 } }`,
expected: "config: {db: {host: \"localhost\", port: 5432}}\n",
scenarioType: "decode",
},
{
description: "mixed list",
skipDoc: true,
input: `values = [1, "two", true]`,
expected: "values:\n - 1\n - \"two\"\n - true\n",
scenarioType: "decode",
},
{
description: "multiple block label keys roundtrip",
description: "Roundtrip: Sample Doc",
input: multipleBlockLabelKeys,
expected: multipleBlockLabelKeysExpected,
scenarioType: "roundtrip",
},
{
description: "multiple block label keys decode",
description: "Roundtrip: With an update",
input: multipleBlockLabelKeys,
expression: `.service.cat.process.main.command += "meow"`,
expected: multipleBlockLabelKeysExpectedUpdate,
scenarioType: "roundtrip",
},
{
description: "Parse HCL: Sample Doc",
input: multipleBlockLabelKeys,
expected: multipleBlockLabelKeysExpectedYaml,
scenarioType: "decode",
},
{
description: "block with labels",
skipDoc: true,
input: `resource "aws_instance" "example" { ami = "ami-12345" }`,
expected: "resource:\n aws_instance:\n example:\n ami: \"ami-12345\"\n",
scenarioType: "decode",
},
{
description: "block with labels roundtrip",
skipDoc: true,
input: `resource "aws_instance" "example" { ami = "ami-12345" }`,
expected: "resource \"aws_instance\" \"example\" {\n ami = \"ami-12345\"\n}\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip simple attribute",
skipDoc: true,
input: `io_mode = "async"`,
expected: `io_mode = "async"` + "\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip number attribute",
skipDoc: true,
input: `port = 8080`,
expected: "port = 8080\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip float attribute",
skipDoc: true,
input: `pi = 3.14`,
expected: "pi = 3.14\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip boolean attribute",
skipDoc: true,
input: `enabled = true`,
expected: "enabled = true\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip list of strings",
skipDoc: true,
input: `tags = ["a", "b"]`,
expected: "tags = [\"a\", \"b\"]\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip object/map attribute",
skipDoc: true,
input: `obj = { a = 1, b = "two" }`,
expected: "obj = {\n a = 1\n b = \"two\"\n}\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip nested block",
skipDoc: true,
input: `server { port = 8080 }`,
expected: "server {\n port = 8080\n}\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip multiple attributes",
skipDoc: true,
input: "name = \"app\"\nversion = 1\nenabled = true",
expected: "name = \"app\"\nversion = 1\nenabled = true\n",
scenarioType: "roundtrip",
},
{
description: "decode with comments",
description: "Parse HCL: with comments",
input: "# Configuration\nport = 8080 # server port",
expected: "# Configuration\nport: 8080 # server port\n",
scenarioType: "decode",
},
{
description: "roundtrip with comments",
description: "Roundtrip: with comments",
input: "# Configuration\nport = 8080",
expected: "# Configuration\nport = 8080\n",
scenarioType: "roundtrip",
},
{
description: "roundtrip example",
input: roundtripSample,
expected: roundtripSampleExpected,
description: "Roundtrip: With templates, functions and arithmetic",
input: simpleSample,
expected: simpleSampleExpected,
scenarioType: "roundtrip",
},
{
description: "roundtrip example",
skipDoc: true,
input: simpleSample,
expected: simpleSampleExpectedYaml,
scenarioType: "decode",
},
{
description: "Parse HCL: List of strings",
skipDoc: true,
input: `tags = ["a", "b"]`,
expected: "tags:\n - \"a\"\n - \"b\"\n",
scenarioType: "decode",
},
}
func testHclScenario(t *testing.T, s formatScenario) {
@@ -285,8 +355,73 @@ func testHclScenario(t *testing.T, s formatScenario) {
}
}
func documentHclScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
s := i.(formatScenario)
if s.skipDoc {
return
}
switch s.scenarioType {
case "", "decode":
documentHclDecodeScenario(w, s)
case "roundtrip":
documentHclRoundTripScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func documentHclDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.hcl file of:\n")
writeOrPanic(w, fmt.Sprintf("```hcl\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if s.expression != "" {
expression = fmt.Sprintf(" '%v'", s.expression)
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy%v sample.hcl\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func documentHclRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.hcl file of:\n")
writeOrPanic(w, fmt.Sprintf("```hcl\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if s.expression != "" {
expression = fmt.Sprintf(" '%v'", s.expression)
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq%v sample.hcl\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```hcl\n%v```\n\n", mustProcessFormatScenario(s, NewHclDecoder(), NewHclEncoder(ConfiguredHclPreferences))))
}
func TestHclFormatScenarios(t *testing.T) {
for _, tt := range hclFormatScenarios {
testHclScenario(t, tt)
}
genericScenarios := make([]interface{}, len(hclFormatScenarios))
for i, s := range hclFormatScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "hcl", genericScenarios, documentHclScenario)
}

View File

@@ -713,10 +713,10 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
if s.expression != "" {
expression = fmt.Sprintf(" '%v'", s.expression)
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression))
writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy%v sample.xml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
@@ -734,7 +734,7 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false '.' sample.xml\n```\n")
writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false sample.xml\n```\n")
writeOrPanic(w, "will output\n")
prefs := NewDefaultXmlPreferences()
prefs.KeepNamespace = false
@@ -758,7 +758,7 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq --xml-raw-token=false '.' sample.xml\n```\n")
writeOrPanic(w, "```bash\nyq --xml-raw-token=false sample.xml\n```\n")
writeOrPanic(w, "will output\n")
prefs := NewDefaultXmlPreferences()
@@ -803,7 +803,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n")
writeOrPanic(w, "```bash\nyq sample.xml\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
@@ -821,7 +821,7 @@ func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq --xml-skip-directives '.' sample.xml\n```\n")
writeOrPanic(w, "```bash\nyq --xml-skip-directives sample.xml\n```\n")
writeOrPanic(w, "will output\n")
prefs := NewDefaultXmlPreferences()
prefs.SkipDirectives = true

View File

@@ -38,6 +38,7 @@ cleanup
cmlu
colorise
colors
coloring
compinit
coolioo
coverprofile