Add lookml-gen (#33)

* proof of concept implementation

* cleaned up the code, added more lookml features

* upd version and readme

* remove binary

* removed supportal linking

* add more error handling and changed formatting

* incremented version

Co-authored-by: Akhat Rakishev <akharakishev@monzo.com>

.gitignore

@@ -0,0 +1,3 @@
+/ddbt_dev
+/.DS_Store
+/.idea

README.md

@@ -41,6 +41,7 @@ ddbt version 0.2.1
 - `ddbt completion zsh` will generate a shell completion script zsh (or bash if you pass that as argument). Detailed steps to set up the completion script can be found in `ddbt completion --help`
 - `ddbt isolate-dag` will create a temporary directory and symlink in all files needed for the given _model_filter_ such that Fishtown's DBT could be run against it without having to be run against every model in your data warehouse
 - `ddbt schema-gen -m my_model` will output a new or updated schema yml file for the model provided in the same directory as the dbt model file.
+- `ddbt lookml-gen my_model` will generate lookml view and copy it to your clipboard
 
 ### Global Arguments
 - `--models model_filter` _or_ `-m model_filter`: Instead of running for every model in your project, DDBT will only execute against the requested models. See filters below for what is accepted for `my_model`

cmd/lookml_gen.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"ddbt/bigquery"
+	"ddbt/config"
+	"errors"
+
+	"fmt"
+	"os"
+
+	"github.com/atotto/clipboard"
+	"github.com/spf13/cobra"
+)
+
+// map bigquery data types to looker data types
+var mapBqToLookerDtypes map[string]string = map[string]string{
+	"INTEGER":   "number",
+	"FLOAT":     "number",
+	"NUMERIC":   "number",
+	"BOOLEAN":   "yesno",
+	"STRING":    "string",
+	"TIMESTAMP": "time",
+	"DATETIME":  "time",
+	"DATE":      "time",
+	"TIME":      "time",
+	"BOOL":      "yesno",
+	"ARRAY":     "string",
+	"GEOGRAPHY": "string",
+}
+
+// specify looker timeframes for datetime/date/time variable data types
+const timeBlock string = `timeframes: [
+  raw,
+  time,
+  date,
+  week,
+  month,
+  quarter,
+  year
+]
+`
+
+func init() {
+	rootCmd.AddCommand(lookmlGenCmd)
+}
+
+var lookmlGenCmd = &cobra.Command{
+	Use:               "lookml-gen [model name]",
+	Short:             "Generates the .view.lkml file for a given model",
+	Args:              cobra.ExactValidArgs(1),
+	ValidArgsFunction: completeModelFn,
+	Run: func(cmd *cobra.Command, args []string) {
+		modelName := args[0]
+
+		// get filesystem, model and target
+		fileSystem, _ := compileAllModels()
+		model := fileSystem.Model(modelName)
+
+		target, err := model.GetTarget()
+		if err != nil {
+			fmt.Println("Could not get target for schema")
+			os.Exit(1)
+		}
+		fmt.Println("\n🎯 Target for retrieving schema:", target.ProjectID+"."+target.DataSet)
+
+		// generate lookml view
+		err = generateNewLookmlView(modelName, target)
+
+		if err != nil {
+			fmt.Println("😒 Something went wrong at lookml view generation: ", err)
+			os.Exit(1)
+		}
+
+	},
+}
+
+func getColumnsForModelWithDtypes(modelName string, target *config.Target) (columns []string, dtypes []string, err error) {
+	schema, err := bigquery.GetColumnsFromTable(modelName, target)
+	if err != nil {
+		fmt.Println("Could not retrieve schema from BigQuery")
+		os.Exit(1)
+	}
+
+	// itereate over fields, record field names and data types
+	for _, fieldSchema := range schema {
+		columns = append(columns, fieldSchema.Name)
+		dtypes = append(dtypes, string(fieldSchema.Type))
+	}
+	return columns, dtypes, err
+}
+
+func generateNewLookmlView(modelName string, target *config.Target) error {
+	bqColumns, bqDtypes, err := getColumnsForModelWithDtypes(modelName, target)
+	if err != nil {
+		fmt.Println("Retrieved BigQuery schema but failed to parse it")
+		os.Exit(1)
+	}
+
+	// initialise lookml view head
+	lookmlView := "view: " + modelName + " {\n\n"
+	lookmlView += "sql_table_name: `" + target.ProjectID + "." + target.DataSet + "." + modelName + "` ;;\n"
+
+	// add dimensions and appropriate blocks for each field
+	for i := 0; i < len(bqColumns); i++ {
+		colName := bqColumns[i]
+		colDtype := mapBqToLookerDtypes[bqDtypes[i]]
+		if colDtype == "" {
+			return errors.New("Did not find Looker data type corresponding to BigQuery data type: " + bqDtypes[i])
+		}
+		newBlock := "\n"
+
+		if colDtype == "date_time" || colDtype == "date" || colDtype == "time" {
+			newBlock += "dimension_group: " + colName + " {\n"
+			newBlock += "type: " + colDtype + "\n"
+			newBlock += timeBlock
+		} else {
+			newBlock += "dimension: " + colName + " {\n"
+			newBlock += "type: " + colDtype + "\n"
+		}
+
+		newBlock += "sql: ${TABLE}." + colName + " ;;\n}\n"
+
+		lookmlView += newBlock
+	}
+
+	// add closing curly bracket and copy to clipboard
+	lookmlView += "}"
+
+	err = clipboard.WriteAll(lookmlView)
+	if err != nil {
+		fmt.Println("Could not write generated LookML to your clipboard")
+		os.Exit(1)
+	}
+	fmt.Println("\n✅ LookML view for " + modelName + " has been copied to your clipboard!")
+
+	return nil
+}

utils/version.go

@@ -1,3 +1,3 @@
 package utils
 
-const DdbtVersion = "0.4.6"
+const DdbtVersion = "0.5.0"