[cmucl-commit] [git] CMU Common Lisp branch file-attribute created. snapshot-2012-06-4-g81f65db
Raymond Toy
rtoy at common-lisp.net
Tue Jun 26 03:47:33 UTC 2012
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CMU Common Lisp".
The branch, file-attribute has been created
at 81f65db452ae1c7c1b8907d10f5bb8bbd4cff37f (commit)
- Log -----------------------------------------------------------------
commit 81f65db452ae1c7c1b8907d10f5bb8bbd4cff37f
Merge: 7ed7451 8a9d1d8
Author: Raymond Toy <toy.raymond at gmail.com>
Date: Mon Jun 25 20:46:20 2012 -0700
Merge branch 'master' into file-attribute
commit 7ed745111ae7a478ba867db257a1888d2948b524
Author: Raymond Toy <toy.raymond at gmail.com>
Date: Thu May 24 21:39:39 2012 -0700
First cut at :file-attribute external format (mostly from Douglas.)
diff --git a/src/code/exports.lisp b/src/code/exports.lisp
index a46d5bb..d0cfe58 100644
--- a/src/code/exports.lisp
+++ b/src/code/exports.lisp
@@ -1511,6 +1511,7 @@
"DESCRIBE-EXTERNAL-FORMAT")
;; Unicode
(:export "STRING-TO-OCTETS" "OCTETS-TO-STRING" "*DEFAULT-EXTERNAL-FORMAT*"
+ "*DEFAULT-SOURCE-EXTERNAL-FORMAT*"
"DESCRIBE-EXTERNAL-FORMAT"
"LIST-ALL-EXTERNAL-FORMATS"
"STRING-ENCODE" "STRING-DECODE"
diff --git a/src/code/fd-stream.lisp b/src/code/fd-stream.lisp
index 45b5847..e854a73 100644
--- a/src/code/fd-stream.lisp
+++ b/src/code/fd-stream.lisp
@@ -1362,6 +1362,169 @@
;;;; Utility functions (misc routines, etc)
+(defvar *stream-encoding-file-attribute-translations*
+ '(;; Emacs specific codings.
+ (:iso8859-1 "latin-1" "latin-1-unix" "iso-latin-1-unix" "iso-8859-1-unix")
+ (:utf-8 "utf-8-unix")
+ (:euc-jp "euc-jp-unix")
+ )
+ "List of coding translations used by 'stream-encoding-file-attribute to map
+ the read file coding into a native external-format. Each element is a list of
+ a native external-format followed byte a list of coding strings that are to be
+ mapped to this native format.")
+
+;;; stream-encoding-file-attribute -- Internal
+;;;
+;;; Read the encoding file option from the stream 's which is expected to be a
+;;; character stream with an external-format of :iso8859-1.
+;;;
+(defun stream-encoding-file-attribute (s)
+ (let* ((initial-encoding nil)
+ (declared-encoding nil)
+ (buffer (make-array 1024 :element-type '(unsigned-byte 8)))
+ (available (do ((i 0 (1+ i)))
+ ((>= i 1024) i)
+ (declare (fixnum i))
+ (let ((ch (read-char s nil nil)))
+ (unless ch (return i))
+ (setf (aref buffer i) (char-code ch))))))
+ (labels ((decode-ascii (start size offset)
+ (declare (type fixnum start)
+ (type (integer 1 4) size)
+ (type (integer 0 3) offset))
+ (let ((ascii (make-array 64 :element-type 'character
+ :adjustable t :fill-pointer 0)))
+ (do ()
+ ((< available (+ start size)))
+ (let* ((code (ecase size
+ (1 (aref buffer start))
+ (2 (let ((b0 (aref buffer start))
+ (b1 (aref buffer (1+ start))))
+ (ecase offset
+ (0 (logior (ash b1 8) b0))
+ (1 (logior (ash b0 8) b1)))))
+ (4
+ (let ((b0 (aref buffer start))
+ (b1 (aref buffer (+ start 1)))
+ (b2 (aref buffer (+ start 2)))
+ (b3 (aref buffer (+ start 3))))
+ (ecase offset
+ (0 (logior (ash b3 24) (ash b2 16) (ash b1 8) b0))
+ (1 (logior (ash b1 24) (ash b0 16) (ash b3 8) b2))
+ (2 (logior (ash b2 24) (ash b3 16) (ash b0 8) b1))
+ (3 (logior (ash b0 24) (ash b1 16) (ash b2 8) b3))))))))
+ (incf start size)
+ (let ((ch (if (< 0 code #x80) (code-char code) #\?)))
+ (vector-push-extend ch ascii))))
+ ascii))
+ (parse-file-option (ascii)
+ ;; Parse the file options.
+ (let ((found (search "-*-" ascii))
+ (options nil))
+ (when found
+ (block do-file-options
+ (let* ((start (+ found 3))
+ (end (search "-*-" ascii :start2 start)))
+ (unless end
+ (return-from do-file-options))
+ (unless (find #\: ascii :start start :end end)
+ (return-from do-file-options))
+ (do ((opt-start start (1+ semi)) colon semi)
+ (nil)
+ (setf colon (position #\: ascii :start opt-start :end end))
+ (unless colon
+ (return-from do-file-options))
+ (setf semi (or (position #\; ascii :start colon :end end) end))
+ (let ((option (string-trim '(#\space #\tab)
+ (subseq ascii opt-start colon)))
+ (value (string-trim '(#\space #\tab)
+ (subseq ascii (1+ colon) semi))))
+ (push (cons option value) options)
+ (when (= semi end) (return nil)))))))
+ (setf declared-encoding
+ (cond ((cdr (assoc "external-format" options :test 'equalp)))
+ ((cdr (assoc "encoding" options :test 'equalp)))
+ ((cdr (assoc "coding" options :test 'equalp))))))))
+ (cond ((>= available 4)
+ (let ((b1 (aref buffer 0))
+ (b2 (aref buffer 1))
+ (b3 (aref buffer 2))
+ (b4 (aref buffer 3)))
+ (cond ((and (= b1 #x00) (= b2 #x00) (= b3 #xFE) (= b4 #xFF))
+ (setf initial-encoding :ucs-4be)
+ (parse-file-option (decode-ascii 4 4 3)))
+ ((and (= b1 #xff) (= b2 #xfe))
+ (cond ((and (= b3 #x00) (= b4 #x00))
+ (setf initial-encoding :ucs-4le)
+ (parse-file-option (decode-ascii 4 4 0)))
+ (t
+ (setf initial-encoding :utf-16le)
+ (parse-file-option (decode-ascii 2 2 0)))))
+ ((and (= b1 #x00) (= b2 #x00) (= b3 #xFF) (= b4 #xFE))
+ (parse-file-option (decode-ascii 4 4 2)))
+ ((and (= b1 #xfe) (= b2 #xff))
+ (cond ((and (= b3 #x00) (= b4 #x00))
+ (parse-file-option (decode-ascii 4 4 1)))
+ (t
+ (setf initial-encoding :utf-16be)
+ (parse-file-option (decode-ascii 2 2 1)))))
+ ((and (= b1 #xEF) (= b2 #xBB) (= b3 #xBF))
+ (setf initial-encoding :utf-8))
+ ((and (> b1 0) (= b2 0) (= b3 0) (= b4 0))
+ (setf initial-encoding :ucs-4le)
+ (parse-file-option (decode-ascii 0 4 0)))
+ ((and (= b1 0) (> b2 0) (= b3 0) (= b4 0))
+ (parse-file-option (decode-ascii 0 4 1)))
+ ((and (= b1 0) (= b2 0) (> b3 0) (= b4 0))
+ (parse-file-option (decode-ascii 0 4 2)))
+ ((and (= b1 0) (= b2 0) (= b3 0) (> b4 0))
+ (setf initial-encoding :ucs-4be)
+ (parse-file-option (decode-ascii 0 4 3)))
+ ((and (> b1 0) (= b2 0) (> b3 0) (= b4 0))
+ (setf initial-encoding :utf-16le)
+ (parse-file-option (decode-ascii 0 2 0)))
+ ((and (= b1 0) (> b2 0) (= b3 0) (> b4 0))
+ (setf initial-encoding :utf-16be)
+ (parse-file-option (decode-ascii 0 2 1)))
+ ((and (= b1 #x2B) (= b2 #x41)
+ (or (= b3 #x43) (= b3 #x44)))
+ (setf initial-encoding :utf-7))
+ ((and (= b1 #x2F) (= b2 #x2B) (= b3 #x41))
+ (setf initial-encoding :utf-7))
+ (t
+ (parse-file-option (decode-ascii 0 1 0))))))
+ ((= available 3)
+ (when (and (= (aref buffer 0) #xEF)
+ (= (aref buffer 1) #xBB)
+ (= (aref buffer 2) #xBF))
+ (setf initial-encoding :utf-8)))
+ ((= available 2)
+ (let ((b1 (aref buffer 0))
+ (b2 (aref buffer 1)))
+ (cond ((and (= b1 #xff) (= b2 #xfe))
+ (setf initial-encoding :utf-16le))
+ ((and (= b1 #xfe) (= b2 #xff))
+ (setf initial-encoding :utf-16be)))))))
+ ;;
+ ;;
+ (cond ((and (not initial-encoding) (not declared-encoding))
+ :default)
+ (t
+ (let ((encoding (or declared-encoding initial-encoding)))
+ (when (stringp encoding)
+ (setf encoding (string-upcase encoding))
+ (dolist (translations *stream-encoding-file-attribute-translations*)
+ (when (member encoding (rest translations) :test 'equalp)
+ (setf encoding (first translations))
+ (return))))
+ (let ((external-format
+ (cond ((eq encoding :default) :default)
+ ((stringp encoding)
+ (intern encoding :keyword))
+ (t
+ encoding))))
+ external-format))))))
+
;;; SET-ROUTINES -- internal
;;;
;;; Fill in the various routine slots for the given type. Input-p and
@@ -1916,20 +2079,7 @@
(setf (fd-stream-flags stream) #b001))
(t
(setf (fd-stream-flags stream) #b010)))
-
- ;; FIXME: setting the external format here should be better
- ;; integrated into set-routines. We do it before so that
- ;; set-routines can create an in-buffer if appropriate. But we
- ;; need to do it after to put the correct input routines for the
- ;; external format.
;;
- ;;#-unicode-bootstrap ; fails in stream-reinit otherwise
- #+(and unicode (not unicode-bootstrap))
- (%set-fd-stream-external-format stream external-format nil)
- (set-routines stream element-type input output input-buffer-p
- :binary-stream-p binary-stream-p)
- #+(and unicode (not unicode-bootstrap))
- (%set-fd-stream-external-format stream external-format nil)
(when (and auto-close (fboundp 'finalize))
(finalize stream
#'(lambda ()
@@ -1937,6 +2087,46 @@
(format *terminal-io* (intl:gettext "** Closed ~A~%") name)
(when original
(revert-file file original)))))
+ ;;
+ ;; FIXME: setting the external format here should be better
+ ;; integrated into set-routines. We do it before so that
+ ;; set-routines can create an in-buffer if appropriate. But we
+ ;; need to do it after to put the correct input routines for the
+ ;; external format.
+ ;;
+ ;;#-unicode-bootstrap ; fails in stream-reinit otherwise
+ #+(and unicode (not unicode-bootstrap))
+ (cond ((and (eq external-format :file-attribute) input)
+ ;; Read the encoding file option with the external-format set to
+ ;; :iso8859-1, and then change the external-format if necessary.
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream :iso8859-1 nil)
+ (set-routines stream element-type input output input-buffer-p
+ :binary-stream-p binary-stream-p)
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream :iso8859-1 nil)
+ (let ((encoding (stream-encoding-file-attribute stream)))
+ (unless (file-position stream :start)
+ (error (intl:gettext "The ~A external-format requires a file stream.")
+ external-format))
+ (unless (member encoding '(:iso8859-1 :iso-8859-1))
+ (setf (stream-external-format stream) (or encoding :default)))))
+ ((eq external-format :file-attribute)
+ ;; Non-input stream, so can not read the file attributes, so use the
+ ;; :default.
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream :default nil)
+ (set-routines stream element-type input output input-buffer-p
+ :binary-stream-p binary-stream-p)
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream :default nil))
+ (t
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream external-format nil)
+ (set-routines stream element-type input output input-buffer-p
+ :binary-stream-p binary-stream-p)
+ #+(and unicode (not unicode-bootstrap))
+ (%set-fd-stream-external-format stream external-format nil)))
stream))
diff --git a/src/code/load.lisp b/src/code/load.lisp
index 832853b..89f7705 100644
--- a/src/code/load.lisp
+++ b/src/code/load.lisp
@@ -19,7 +19,7 @@
(in-package "EXTENSIONS")
(export '(*load-if-source-newer* *load-source-types* *load-object-types*
- invalid-fasl))
+ invalid-fasl *default-source-external-format*))
(in-package "SYSTEM")
(export '(foreign-symbol-address alternate-get-global-address))
@@ -94,6 +94,12 @@
(invalid-fasl-pathname condition)
(invalid-fasl-version condition)
(invalid-fasl-expected-version condition)))))
+
+(defvar *default-source-external-format* :default
+ "The external-format that 'load and 'compile-file use when given an
+ external-format of :default. The default value is :default which will open
+ the file using the 'ext:*default-external-format*")
+
;;; LOAD-FRESH-LINE -- internal.
;;;
@@ -523,6 +529,10 @@
defaulting. Probably only necessary if you have source files with a
\"fasl\" type.
+ :EXTERNAL-FORMAT
+ The external-format to use when opening the FILENAME. The default is
+ :default which uses the EXT:*DEFAULT-SOURCE-EXTERNAL-FORMAT*.
+
The variables *LOAD-VERBOSE*, *LOAD-PRINT* and EXT:*LOAD-IF-SOURCE-NEWER*
determine the defaults for the corresponding keyword arguments. These
variables are also bound to the specified argument values, so specifying a
@@ -604,6 +614,8 @@
(*load-pathname* pathname))
(case contents
(:source
+ (when (eq external-format :default)
+ (setf external-format *default-source-external-format*))
(with-open-file (file truename :external-format external-format
:direction :input
:if-does-not-exist if-does-not-exist)
diff --git a/src/compiler/main.lisp b/src/compiler/main.lisp
index 544ccb0..066ff2d 100644
--- a/src/compiler/main.lisp
+++ b/src/compiler/main.lisp
@@ -738,12 +738,12 @@
:write-date (file-write-date x)
:language :lisp))
files)))
-
+ (when (eq external-format :default)
+ (setf external-format *default-source-external-format*))
(make-source-info :files file-info
:current-file file-info
#+unicode :external-format
- #+unicode (stream::ef-name
- (stream::find-external-format external-format))
+ #+unicode external-format
#+unicode :decoding-error
#+unicode decoding-error)))
diff --git a/src/i18n/locale/cmucl.pot b/src/i18n/locale/cmucl.pot
index 5b38108..bb807f4 100644
--- a/src/i18n/locale/cmucl.pot
+++ b/src/i18n/locale/cmucl.pot
@@ -9187,6 +9187,16 @@ msgid "Error reading ~S: ~A"
msgstr ""
#: src/code/fd-stream.lisp
+msgid ""
+"List of coding translations used by 'stream-encoding-file-attribute to map\n"
+" the read file coding into a native external-format. Each element is a "
+"list of\n"
+" a native external-format followed byte a list of coding strings that are "
+"to be\n"
+" mapped to this native format."
+msgstr ""
+
+#: src/code/fd-stream.lisp
msgid "Could not find any input routine for ~S"
msgstr ""
@@ -9263,6 +9273,10 @@ msgid "** Closed ~A~%"
msgstr ""
#: src/code/fd-stream.lisp
+msgid "The ~A external-format requires a file stream."
+msgstr ""
+
+#: src/code/fd-stream.lisp
msgid ""
"This is a string that OPEN tacks on the end of a file namestring to produce\n"
" a name for the :if-exists :rename-and-delete and :rename options. Also,\n"
@@ -10064,6 +10078,14 @@ msgid ""
msgstr ""
#: src/code/load.lisp
+msgid ""
+"The external-format that 'load and 'compile-file use when given an\n"
+" external-format of :default. The default value is :default which will "
+"open\n"
+" the file using the 'ext:*default-external-format*"
+msgstr ""
+
+#: src/code/load.lisp
msgid "List of free fop tables for the fasloader."
msgstr ""
@@ -10133,6 +10155,10 @@ msgid ""
" defaulting. Probably only necessary if you have source files with a\n"
" \"fasl\" type. \n"
"\n"
+" :EXTERNAL-FORMAT\n"
+" The external-format to use when opening the FILENAME. The default is\n"
+" :default which uses the EXT:*DEFAULT-SOURCE-EXTERNAL-FORMAT*.\n"
+"\n"
" The variables *LOAD-VERBOSE*, *LOAD-PRINT* and EXT:*LOAD-IF-SOURCE-NEWER"
"*\n"
" determine the defaults for the corresponding keyword arguments. These\n"
-----------------------------------------------------------------------
hooks/post-receive
--
CMU Common Lisp
More information about the cmucl-commit
mailing list