gh-47005: fix do_open() to let regular headers override unredirected headers

AbstractHTTPHandler.do_open() was building the request header dict by
starting with unredirected_hdrs and only inserting regular headers that
were not already present, giving unredirected headers priority.  This
contradicts get_header() and header_items(), both of which give regular
headers the higher priority.

Fix by unconditionally updating with req.headers so that a header set
via add_header() always overrides one set via add_unredirected_header().

Author: Das-Chinmay

Lib/test/test_urllib2.py

@@ -980,6 +980,23 @@ def test_http(self):
             self.assertEqual(req.unredirected_hdrs["Host"], "baz")
             self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
 
+    def test_http_header_priority(self):
+        # gh-47005: regular headers set via add_header() must override
+        # unredirected headers with the same name in do_open(), consistent
+        # with get_header() and header_items().
+        h = urllib.request.AbstractHTTPHandler()
+        h.parent = MockOpener()
+
+        req = Request("http://example.com/", headers={"Content-Type": "application/json"})
+        req.timeout = None
+        req.add_unredirected_header("Content-Type", "application/x-www-form-urlencoded")
+
+        http = MockHTTPClass()
+        h.do_open(http, req)
+
+        sent_headers = dict(http.req_headers)
+        self.assertEqual(sent_headers["Content-Type"], "application/json")
+
     def test_http_body_file(self):
         # A regular file - chunked encoding is used unless Content Length is
         # already set.

Lib/urllib/request.py

@@ -1293,8 +1293,7 @@ def do_open(self, http_class, req, **http_conn_args):
         h.set_debuglevel(self._debuglevel)
 
         headers = dict(req.unredirected_hdrs)
-        headers.update({k: v for k, v in req.headers.items()
-                        if k not in headers})
+        headers.update(req.headers)
 
         # TODO(jhylton): Should this be redesigned to handle
         # persistent connections?