diff --git a/.config/nvim/lua/opts.lua b/.config/nvim/lua/opts.lua
index f1b1fd4f..1ded4a97 100644
--- a/.config/nvim/lua/opts.lua
+++ b/.config/nvim/lua/opts.lua
@@ -6,7 +6,6 @@ vim.o.ignorecase = true
 vim.o.smartcase = true
 
 vim.o.number = true
-vim.o.relativenumber = true
 vim.o.signcolumn = "yes"
 
 vim.o.expandtab = true
diff --git a/.config/nvim/plugin/autocommands.lua b/.config/nvim/plugin/autocommands.lua
index 3dcbf999..e4bf06a0 100644
--- a/.config/nvim/plugin/autocommands.lua
+++ b/.config/nvim/plugin/autocommands.lua
@@ -28,7 +28,7 @@ vim.api.nvim_create_autocmd("OptionSet", {
     pattern = "textwidth",
     callback = set_colorcolumn_from_textwidth,
 })
-vim.api.nvim_create_autocmd("BufEnter", {
+vim.api.nvim_create_autocmd("BufWinEnter", {
     group = auto_colorcolumn,
     callback = function()
         if vim.bo.textwidth then
@@ -53,3 +53,20 @@ vim.api.nvim_create_autocmd("WinLeave", {
     group = auto_cursorline,
     callback = function() vim.wo.cursorline = false end,
 })
+
+local auto_linenr = vim.api.nvim_create_augroup("AutoLineNr", {})
+vim.api.nvim_create_autocmd({ "VimEnter", "WinEnter", "BufWinEnter" }, {
+    group = auto_linenr,
+    callback = function()
+        -- fugitive, for example, doesn't have numbers
+        -- if I set relative number I'll get numbers and we don't want that
+        if vim.wo.number then
+            vim.wo.relativenumber = true
+        end
+    end,
+})
+
+vim.api.nvim_create_autocmd("WinLeave", {
+    group = auto_linenr,
+    callback = function() vim.wo.relativenumber = false end,
+})